diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml new file mode 100644 index 00000000000000..cbaf8019215cdc --- /dev/null +++ b/.github/workflows/mingw.yml @@ -0,0 +1,294 @@ +name: Build +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + msystem: ['MINGW64','MINGW32','UCRT64','CLANG64','CLANG32'] + include: + - msystem: MINGW64 + prefix: mingw-w64-x86_64 + - msystem: MINGW32 + prefix: mingw-w64-i686 + - msystem: UCRT64 + prefix: mingw-w64-ucrt-x86_64 + - msystem: CLANG64 + prefix: mingw-w64-clang-x86_64 + - msystem: CLANG32 + prefix: mingw-w64-clang-i686 + steps: + - name: Setup git + run: | + git config --global core.autocrlf false + git config --global core.eol lf + - uses: actions/checkout@v3 + - uses: msys2/setup-msys2@v2 + with: + msystem: ${{ matrix.msystem }} + release: false + update: true + install: >- + make + binutils + autoconf + autoconf-archive + automake-wrapper + tar + gzip + ${{ matrix.prefix }}-toolchain + ${{ matrix.prefix }}-expat + ${{ matrix.prefix }}-bzip2 + ${{ matrix.prefix }}-libffi + ${{ matrix.prefix }}-mpdecimal + ${{ matrix.prefix }}-ncurses + ${{ matrix.prefix }}-openssl + ${{ matrix.prefix }}-sqlite3 + ${{ matrix.prefix }}-tcl + ${{ matrix.prefix }}-tk + ${{ matrix.prefix }}-zlib + ${{ matrix.prefix }}-xz + ${{ matrix.prefix }}-tzdata + + - name: Build Python + shell: msys2 {0} + run: | + set -ex + + if [[ "${{ matrix.msystem }}" == "CLANG64" ]] || [[ "${{ matrix.msystem }}" == "CLANG32" ]]; then + export CC=clang + export CXX=clang++ + fi + autoreconf -vfi + + rm -Rf _build && mkdir _build && cd _build + + ../configure \ + --prefix=${MINGW_PREFIX} \ + --host=${MINGW_CHOST} \ + --build=${MINGW_CHOST} \ + --enable-shared \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-tzpath=${MINGW_PREFIX}/share/zoneinfo \ + --enable-optimizations + + make -j8 + + - name: Run Smoke Test (build) + shell: msys2 {0} + run: | + export SETUPTOOLS_USE_DISTUTILS=stdlib + SMOKETESTS="$(pwd)/mingw_smoketests.py" + cd _build + ./python.exe "$SMOKETESTS" + MSYSTEM= ./python.exe "$SMOKETESTS" + + - name: Run tests + shell: msys2 {0} + run: | + export SETUPTOOLS_USE_DISTUTILS=stdlib + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --ignorefile "$IGNOREFILE" -W + + - name: Run broken tests + continue-on-error: true + shell: msys2 {0} + run: | + export SETUPTOOLS_USE_DISTUTILS=stdlib + IGNOREFILE="$(pwd)/mingw_ignorefile.txt" + cd _build + MSYSTEM= ./python.exe -m test -j8 --matchfile "$IGNOREFILE" -W + + - name: Install + shell: msys2 {0} + run: | + set -ex + cd _build + + pkgdir=python_pkgdir + + make -j1 install DESTDIR="${pkgdir}" + + # Fix shebangs + _pybasever=$(./python.exe -c "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}');") + for fscripts in 2to3 2to3-${_pybasever} idle3 idle${_pybasever} pydoc3 pydoc${_pybasever}; do + sed -i "s|$(cygpath -w ${MINGW_PREFIX} | sed 's|\\|\\\\|g')/bin/python${_pybasever}.exe|/usr/bin/env python${_pybasever}.exe|g" "${pkgdir}${MINGW_PREFIX}"/bin/${fscripts} + done + sed -i "s|#!${MINGW_PREFIX}/bin/python${_pybasever}.exe|#!/usr/bin/env python${_pybasever}.exe|" "${pkgdir}${MINGW_PREFIX}"/lib/python${_pybasever}/config-${_pybasever}/python-config.py + + # Create version-less aliases + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3.exe "${pkgdir}${MINGW_PREFIX}"/bin/python.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3w.exe "${pkgdir}${MINGW_PREFIX}"/bin/pythonw.exe + cp "${pkgdir}${MINGW_PREFIX}"/bin/python3-config "${pkgdir}${MINGW_PREFIX}"/bin/python-config + cp "${pkgdir}${MINGW_PREFIX}"/bin/idle3 "${pkgdir}${MINGW_PREFIX}"/bin/idle + cp "${pkgdir}${MINGW_PREFIX}"/bin/pydoc3 "${pkgdir}${MINGW_PREFIX}"/bin/pydoc + + # copy to / + cp -rf "${pkgdir}"/* / + + - name: Run Smoke Test (installed) + shell: msys2 {0} + run: | + export SETUPTOOLS_USE_DISTUTILS=stdlib + SMOKETESTS="$(pwd)/mingw_smoketests.py" + ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" + MSYSTEM= ${MINGW_PREFIX}/bin/python.exe "$SMOKETESTS" + + - name: Compress + if: always() + shell: msys2 {0} + run: | + cd _build + tar -zcf python.tar.gz python_pkgdir/ + + - name: Upload + uses: actions/upload-artifact@v3 + if: always() + with: + name: build-${{ matrix.msystem }} + path: _build/python.tar.gz + + cross-gcc-x86_64: + runs-on: ubuntu-latest + container: + image: archlinux:base-devel + steps: + - uses: actions/checkout@v3 + - name: Install deps + run: | + pacman --noconfirm -Suuy + pacman --needed --noconfirm -S mingw-w64-gcc autoconf-archive autoconf automake zip + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Check Python Version + run: | + which python + python --version + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + ../configure \ + --host=x86_64-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ + --without-ensurepip \ + --enable-loadable-sqlite-extensions \ + --with-build-python=python3.11 + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-gcc-x86_64 + path: install.zip + + cross-gcc-x86_64-test: + needs: [cross-gcc-x86_64] + runs-on: windows-latest + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-gcc-x86_64 + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + + cross-llvm-mingw: + runs-on: ubuntu-latest + container: + image: mstorsjo/llvm-mingw:latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686', 'aarch64', 'armv7'] + steps: + - uses: actions/checkout@v3 + + - name: Install deps + run: | + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -qqy software-properties-common + add-apt-repository --yes ppa:deadsnakes/ppa + apt-get update -qq + apt-get install -qqy autoconf-archive python3.11-dev python3.11 + + - name: Build + run: | + autoreconf -vfi + + mkdir _build && cd _build + + export CC=${{ matrix.arch }}-w64-mingw32-clang + export CXX=${CC}++ + ../configure \ + --host=${{ matrix.arch }}-w64-mingw32 \ + --build=x86_64-pc-linux-gnu \ + --enable-shared \ + --with-system-expat \ + --with-system-ffi \ + --with-system-libmpdec \ + --without-ensurepip \ + --without-c-locale-coercion \ + --enable-loadable-sqlite-extensions \ + --with-build-python=python3.11 + + make -j8 + + make install DESTDIR="$(pwd)/install" + + - name: 'Zip files' + run: | + zip -r install.zip _build/install + + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + path: install.zip + + cross-llvm-mingw-test: + needs: [cross-llvm-mingw] + runs-on: windows-latest + strategy: + fail-fast: false + matrix: + arch: ['x86_64', 'i686'] + steps: + - uses: actions/download-artifact@v3 + with: + name: build-cross-llvm-mingw-${{ matrix.arch }} + + - name: 'Run tests' + run: | + 7z x install.zip + ./_build/install/usr/local/bin/python3.exe -c "import sysconfig, pprint; pprint.pprint(sysconfig.get_config_vars())" + + diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 4c4dc40d705d71..cd88bed1d571f6 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -35,9 +35,9 @@ PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); + Py_PRINTF(1, 0); PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); diff --git a/Include/fileobject.h b/Include/fileobject.h index 4c983e7b5daa8a..260e4c111b2cd3 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -30,7 +30,7 @@ PyAPI_DATA(int) Py_UTF8Mode; #endif /* A routine to check if a file descriptor can be select()-ed. */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS /* On Windows, any socket fd can be select()-ed, no matter how high */ #define _PyIsSelectable_fd(FD) (1) #else diff --git a/Include/internal/pycore_condvar.h b/Include/internal/pycore_condvar.h index 981c962bf7dfdf..ed9e6a7cc95f24 100644 --- a/Include/internal/pycore_condvar.h +++ b/Include/internal/pycore_condvar.h @@ -5,6 +5,12 @@ # error "this header requires Py_BUILD_CORE define" #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + #ifndef _POSIX_THREADS /* This means pthreads are not implemented in libc headers, hence the macro not present in unistd.h. But they still can be implemented as an external @@ -39,6 +45,10 @@ /* include windows if it hasn't been done before */ #define WIN32_LEAN_AND_MEAN #include +/* winpthreads are involved via windows header, so need undef _POSIX_THREADS after header include */ +#if defined(_POSIX_THREADS) +#undef _POSIX_THREADS +#endif /* options */ /* non-emulated condition variables are provided for those that want diff --git a/Include/iscygpty.h b/Include/iscygpty.h new file mode 100644 index 00000000000000..82fd0affbd4875 --- /dev/null +++ b/Include/iscygpty.h @@ -0,0 +1,41 @@ +/* + * iscygpty.h -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _ISCYGPTY_H +#define _ISCYGPTY_H + +#ifdef _WIN32 +int is_cygpty(int fd); +int is_cygpty_used(void); +#else +#define is_cygpty(fd) 0 +#define is_cygpty_used() 0 +#endif + +#endif /* _ISCYGPTY_H */ diff --git a/Include/osdefs.h b/Include/osdefs.h index 3243944a1483e9..99d49777a69d9a 100644 --- a/Include/osdefs.h +++ b/Include/osdefs.h @@ -10,7 +10,6 @@ extern "C" { #ifdef MS_WINDOWS #define SEP L'\\' #define ALTSEP L'/' -#define MAXPATHLEN 256 #define DELIM L';' #endif diff --git a/Include/py_curses.h b/Include/py_curses.h index b2c7f1bb4309c6..e6fc813443158f 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,13 @@ #include #endif +#if defined(__MINGW32__) +#include +#if !defined(_ISPAD) +#define _ISPAD 0x10 +#endif +#endif + #ifdef HAVE_NCURSES_H /* configure was checking , but we will use , which has some or all these features. */ diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 34e3de3328f410..fb71fde2163ccb 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -315,9 +315,9 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason( ); PyAPI_FUNC(int) PyOS_snprintf(char *str, size_t size, const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 3, 4))); + Py_PRINTF(3, 4); PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) - Py_GCC_ATTRIBUTE((format(printf, 3, 0))); + Py_PRINTF(3, 0); #ifndef Py_LIMITED_API # define Py_CPYTHON_ERRORS_H diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index e4c3b09c963fe8..21346baf73e2ca 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -21,6 +21,15 @@ PyAPI_FUNC(int) Py_IsInitialized(void); PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void); PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *); +PyAPI_FUNC(wchar_t) Py_GetAltSepW(const wchar_t *); +PyAPI_FUNC(wchar_t) Py_GetSepW(const wchar_t *); +PyAPI_FUNC(char) Py_GetSepA(const char *); + +PyAPI_FUNC(void) Py_NormalizeSepsW(wchar_t *); +PyAPI_FUNC(void) Py_NormalizeSepsA(char *); + +PyAPI_FUNC(void) Py_NormalizeSepsPathcchW(wchar_t *); + /* Py_PyAtExit is for the atexit module, Py_AtExit is for low-level * exit functions. diff --git a/Include/pyport.h b/Include/pyport.h index 93250f4eb1d7a2..b816c90e038686 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -53,6 +53,21 @@ #endif +#ifdef __MINGW32__ +/* Translate GCC[mingw*] platform specific defines to those + * used in python code. + */ +#if !defined(MS_WIN64) && defined(_WIN64) +# define MS_WIN64 +#endif +#if !defined(MS_WIN32) && defined(_WIN32) +# define MS_WIN32 +#endif +#if !defined(MS_WINDOWS) && defined(MS_WIN32) +# define MS_WINDOWS +#endif +#endif /* __MINGW32__*/ + /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -509,12 +524,12 @@ extern char * _getpty(int *, int, mode_t, int); */ /* - All windows ports, except cygwin, are handled in PC/pyconfig.h. + Only MSVC windows ports is handled in PC/pyconfig.h. - Cygwin is the only other autoconf platform requiring special + Cygwin and Mingw is the only other autoconf platform requiring special linkage handling and it uses __declspec(). */ -#if defined(__CYGWIN__) +#if defined(__CYGWIN__) || defined(__MINGW32__) # define HAVE_DECLSPEC_DLL #endif @@ -527,21 +542,23 @@ extern char * _getpty(int *, int, mode_t, int); # define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE # define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE /* module init functions inside the core need no external linkage */ - /* except for Cygwin to handle embedding */ -# if defined(__CYGWIN__) + /* except for Cygwin/Mingw to handle embedding */ +# if defined(__CYGWIN__) || defined(__MINGW32__) # define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject* -# else /* __CYGWIN__ */ +# else /* __CYGWIN__ || __MINGW32__*/ # define PyMODINIT_FUNC PyObject* -# endif /* __CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # else /* Py_BUILD_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ /* failures similar to those described at the bottom of 4.1: */ /* http://docs.python.org/extending/windows.html#a-cookbook-approach */ -# if !defined(__CYGWIN__) +# if defined(__CYGWIN__) || defined(__MINGW32__) +# define PyAPI_FUNC(RTYPE) RTYPE +# else # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE -# endif /* !__CYGWIN__ */ +# endif /* __CYGWIN__ || __MINGW32__*/ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE /* module init functions outside the core must be exported */ # if defined(__cplusplus) @@ -641,6 +658,12 @@ extern char * _getpty(int *, int, mode_t, int); #define Py_VA_COPY va_copy +#if defined(__GNUC__) && ((__GNUC__ == 4 && __GNUC_MINOR__>= 4) || __GNUC__ > 4) +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(gnu_printf,X,Y))) +#else +# define Py_PRINTF(X,Y) Py_GCC_ATTRIBUTE((format(printf,X,Y))) +#endif + /* * Convenient macros to deal with endianness of the platform. WORDS_BIGENDIAN is * detected by configure and defined in pyconfig.h. The code in pyconfig.h diff --git a/Include/pythread.h b/Include/pythread.h index a48329085fac51..9bf8bd61ff3749 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -7,6 +7,12 @@ typedef void *PyThread_type_lock; extern "C" { #endif +#ifdef __MINGW32__ +# if !defined(HAVE_PTHREAD_H) || defined(NT_THREADS) +# undef _POSIX_THREADS +# endif +#endif + /* Return status codes for Python lock acquisition. Chosen for maximum * backwards compatibility, ie failure -> 0, success -> 1. */ typedef enum PyLockStatus { diff --git a/Include/sysmodule.h b/Include/sysmodule.h index b5087119b1cae7..d6767dc2466c81 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -15,9 +15,9 @@ Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetArgvEx(int, wchar_t **, int); Py_DEPRECATED(3.11) PyAPI_FUNC(void) PySys_SetPath(const wchar_t *); PyAPI_FUNC(void) PySys_WriteStdout(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); + Py_PRINTF(1, 2); PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); diff --git a/Lib/compileall.py b/Lib/compileall.py index a388931fb5a99d..069ea2b6cb8ee3 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -38,6 +38,8 @@ def _walk_dir(dir, maxlevels, quiet=0): if name == '__pycache__': continue fullname = os.path.join(dir, name) + if sys.platform == "win32" and sys.version.find("GCC") >= 0: + fullname = fullname.replace('\\','/') if not os.path.isdir(fullname): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index 26135ad96296ac..76d583bbe3f540 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -458,7 +458,9 @@ def LoadLibrary(self, name): cdll = LibraryLoader(CDLL) pydll = LibraryLoader(PyDLL) -if _os.name == "nt": +if _os.name == "nt" and _sys.version.find('GCC') >= 0: + pythonapi = PyDLL("libpython%d.%d%s.dll" % (_sys.version_info[:2] + (_sys.abiflags,)), None) +elif _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) elif _sys.platform == "cygwin": pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 0c2510e1619c8e..48ddb3beb13c87 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -31,6 +31,12 @@ def _get_build_version(): # else we don't know what version of the compiler this is return None + def find_msvcrt_mingw(): + is_ucrt = 'clang' in sys.version.lower() or 'ucrt' in sys.version.lower() + if is_ucrt: + return None + return 'msvcrt.dll' + def find_msvcrt(): """Return the name of the VC runtime dll""" version = _get_build_version() @@ -54,6 +60,9 @@ def find_msvcrt(): def find_library(name): if name in ('c', 'm'): + import sysconfig + if sysconfig.get_platform().startswith('mingw'): + return find_msvcrt_mingw() return find_msvcrt() # See MSDN for the REAL search order. for directory in os.environ['PATH'].split(os.pathsep): diff --git a/Lib/distutils/ccompiler.py b/Lib/distutils/ccompiler.py index 4c47f2ed245d4f..ab61a867292890 100644 --- a/Lib/distutils/ccompiler.py +++ b/Lib/distutils/ccompiler.py @@ -9,7 +9,7 @@ from distutils.file_util import move_file from distutils.dir_util import mkpath from distutils.dep_util import newer_group -from distutils.util import split_quoted, execute +from distutils.util import split_quoted, execute, get_platform from distutils import log class CCompiler: @@ -948,6 +948,8 @@ def get_default_compiler(osname=None, platform=None): osname = os.name if platform is None: platform = sys.platform + if get_platform().startswith('mingw'): + return 'mingw32' for pattern, compiler in _default_compilers: if re.match(pattern, platform) is not None or \ re.match(pattern, osname) is not None: diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index f287b349984ff0..e87e3ad5d4cbb4 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -186,7 +186,7 @@ def finalize_options(self): # for extensions under windows use different directories # for Release and Debug builds. # also Python's library directory must be appended to library_dirs - if os.name == 'nt': + if os.name == 'nt' and not self.plat_name.startswith(('mingw')): # the 'libs' directory is for binary installs - we assume that # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. @@ -218,15 +218,16 @@ def finalize_options(self): # For extensions under Cygwin, Python's library directory must be # appended to library_dirs - if sys.platform[:6] == 'cygwin': - if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + if sys.platform[:6] == 'cygwin' or self.plat_name.startswith(('mingw')): + if not sysconfig.python_build: # building third party extensions + config_dir_name = os.path.basename(sysconfig.get_config_var('LIBPL')) self.library_dirs.append(os.path.join(sys.prefix, "lib", "python" + get_python_version(), - "config")) + config_dir_name)) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(sysconfig.project_base) # For building extensions with a shared Python library, # Python's library directory must be appended to library_dirs @@ -237,7 +238,7 @@ def finalize_options(self): self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) else: # building python standard extensions - self.library_dirs.append('.') + self.library_dirs.append(sysconfig.project_base) # The argument parsing will result in self.define being a string, but # it has to be a list of 2-tuples. All the preprocessor symbols @@ -712,6 +713,20 @@ def get_libraries(self, ext): # pyconfig.h that MSVC groks. The other Windows compilers all seem # to need it mentioned explicitly, though, so that's what we do. # Append '_d' to the python import library on debug builds. + + # Use self.plat_name as it works even in case of + # cross-compilation (at least for mingw build). + if self.plat_name.startswith('mingw'): + from distutils import sysconfig + extra = [] + for lib in ( + sysconfig.get_config_var('BLDLIBRARY').split() + + sysconfig.get_config_var('SHLIBS').split() + ): + if lib.startswith('-l'): + extra.append(lib[2:]) + return ext.libraries + extra + if sys.platform == "win32": from distutils._msvccompiler import MSVCCompiler if not isinstance(self.compiler, MSVCCompiler): diff --git a/Lib/distutils/command/install.py b/Lib/distutils/command/install.py index 01d5331a63069b..25eb3d8d10133a 100644 --- a/Lib/distutils/command/install.py +++ b/Lib/distutils/command/install.py @@ -72,8 +72,8 @@ INSTALL_SCHEMES['nt_user'] = { 'purelib': '$usersite', 'platlib': '$usersite', - 'headers': '$userbase/Python$py_version_nodot/Include/$dist_name', - 'scripts': '$userbase/Python$py_version_nodot/Scripts', + 'headers': '$userbase/include/python$py_version_short_plat$abiflags/$dist_name', + 'scripts': '$userbase/bin', 'data' : '$userbase', } @@ -81,7 +81,7 @@ 'purelib': '$usersite', 'platlib': '$usersite', 'headers': - '$userbase/include/python$py_version_short$abiflags/$dist_name', + '$userbase/include/python$py_version_short_plat$abiflags/$dist_name', 'scripts': '$userbase/bin', 'data' : '$userbase', } @@ -311,6 +311,7 @@ def finalize_options(self): 'py_version': py_version, 'py_version_short': '%d.%d' % sys.version_info[:2], 'py_version_nodot': '%d%d' % sys.version_info[:2], + 'py_version_short_plat': f'{sys.version_info[0]}.{sys.version_info[1]}-{get_platform()}' if os.name == 'nt' and 'gcc' in sys.version.lower() else f'{sys.version_info[0]}.{sys.version_info[1]}', 'sys_prefix': prefix, 'prefix': prefix, 'sys_exec_prefix': exec_prefix, @@ -363,7 +364,8 @@ def finalize_options(self): # Convert directories from Unix /-separated syntax to the local # convention. - self.convert_paths('lib', 'purelib', 'platlib', + self.convert_paths('base', 'platbase', + 'lib', 'purelib', 'platlib', 'scripts', 'data', 'headers') if HAS_USER_SITE: self.convert_paths('userbase', 'usersite') diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py index 66c12dd35830b2..d8c8428ac0d440 100644 --- a/Lib/distutils/cygwinccompiler.py +++ b/Lib/distutils/cygwinccompiler.py @@ -44,12 +44,13 @@ # (ld supports -shared) # * mingw gcc 3.2/ld 2.13 works # (ld supports -shared) +# * llvm-mingw with Clang 11 works +# (lld supports -shared) import os import sys import copy -from subprocess import Popen, PIPE, check_output -import re +import shlex from distutils.unixccompiler import UnixCCompiler from distutils.file_util import write_file @@ -57,6 +58,7 @@ CompileError, UnknownFileError) from distutils.version import LooseVersion from distutils.spawn import find_executable +from subprocess import Popen, check_output def get_msvcr(): """Include the appropriate MSVC runtime library if Python was built @@ -91,6 +93,7 @@ class CygwinCCompiler(UnixCCompiler): obj_extension = ".o" static_lib_extension = ".a" shared_lib_extension = ".dll" + dylib_lib_extension = ".dll.a" static_lib_format = "lib%s%s" shared_lib_format = "%s%s" exe_extension = ".exe" @@ -109,50 +112,28 @@ def __init__(self, verbose=0, dry_run=0, force=0): "Compiling may fail because of undefined preprocessor macros." % details) - self.gcc_version, self.ld_version, self.dllwrap_version = \ - get_versions() - self.debug_print(self.compiler_type + ": gcc %s, ld %s, dllwrap %s\n" % - (self.gcc_version, - self.ld_version, - self.dllwrap_version) ) - - # ld_version >= "2.10.90" and < "2.13" should also be able to use - # gcc -mdll instead of dllwrap - # Older dllwraps had own version numbers, newer ones use the - # same as the rest of binutils ( also ld ) - # dllwrap 2.10.90 is buggy - if self.ld_version >= "2.10.90": - self.linker_dll = "gcc" - else: - self.linker_dll = "dllwrap" + self.cc = os.environ.get('CC', 'gcc') + self.cxx = os.environ.get('CXX', 'g++') - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # Hard-code GCC because that's what this is all about. - # XXX optimization, warnings etc. should be customizable. - self.set_executables(compiler='gcc -mcygwin -O -Wall', - compiler_so='gcc -mcygwin -mdll -O -Wall', - compiler_cxx='g++ -mcygwin -O -Wall', - linker_exe='gcc -mcygwin', + # Older numpy dependend on this existing to check for ancient + # gcc versions. This doesn't make much sense with clang etc so + # just hardcode to something recent. + # https://github.com/numpy/numpy/pull/20333 + self.gcc_version = LooseVersion("11.2.0") + + self.linker_dll = self.cc + shared_option = "-shared" + + self.set_executables(compiler='%s -mcygwin -O -Wall' % self.cc, + compiler_so='%s -mcygwin -mdll -O -Wall' % self.cc, + compiler_cxx='%s -mcygwin -O -Wall' % self.cxx, + linker_exe='%s -mcygwin' % self.cc, linker_so=('%s -mcygwin %s' % (self.linker_dll, shared_option))) - # cygwin and mingw32 need different sets of libraries - if self.gcc_version == "2.91.57": - # cygwin shouldn't need msvcrt, but without the dlls will crash - # (gcc version 2.91.57) -- perhaps something about initialization - self.dll_libraries=["msvcrt"] - self.warn( - "Consider upgrading to a newer version of gcc") - else: - # Include the appropriate MSVC runtime library if Python was built - # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() + # Include the appropriate MSVC runtime library if Python was built + # with MSVC 7.0 or later. + self.dll_libraries = get_msvcr() def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): """Compiles the source by spawning GCC and windres if needed.""" @@ -162,6 +143,28 @@ def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): self.spawn(["windres", "-i", src, "-o", obj]) except DistutilsExecError as msg: raise CompileError(msg) + elif ext == '.mc': + # Adapted from msvc9compiler: + # + # Compile .MC to .RC file to .RES file. + # * '-h dir' specifies the directory for the generated include file + # * '-r dir' specifies the target directory of the generated RC file and the binary message resource it includes + # + # For now (since there are no options to change this), + # we use the source-directory for the include file and + # the build directory for the RC file and message + # resources. This works at least for win32all. + h_dir = os.path.dirname(src) + rc_dir = os.path.dirname(obj) + try: + # first compile .MC to .RC and .H file + self.spawn(['windmc'] + ['-h', h_dir, '-r', rc_dir] + [src]) + base, _ = os.path.splitext(os.path.basename(src)) + rc_file = os.path.join(rc_dir, base + '.rc') + # then compile .RC to .RES file + self.spawn(['windres', '-i', rc_file, '-o', obj]) + except DistutilsExecError as msg: + raise CompileError(msg) else: # for other files use the C-compiler try: self.spawn(self.compiler_so + cc_args + [src, '-o', obj] + @@ -214,28 +217,21 @@ def link(self, target_desc, objects, output_filename, output_dir=None, # next add options for def-file and to creating import libraries - # dllwrap uses different options than gcc/ld - if self.linker_dll == "dllwrap": - extra_preargs.extend(["--output-lib", lib_file]) - # for dllwrap we have to use a special option - extra_preargs.extend(["--def", def_file]) - # we use gcc/ld here and can be sure ld is >= 2.9.10 - else: - # doesn't work: bfd_close build\...\libfoo.a: Invalid operation - #extra_preargs.extend(["-Wl,--out-implib,%s" % lib_file]) - # for gcc/ld the def-file is specified as any object files - objects.append(def_file) + # doesn't work: bfd_close build\...\libfoo.a: Invalid operation + #extra_preargs.extend(["-Wl,--out-implib,%s" % lib_file]) + # for gcc/ld the def-file is specified as any object files + objects.append(def_file) #end: if ((export_symbols is not None) and # (target_desc != self.EXECUTABLE or self.linker_dll == "gcc")): # who wants symbols and a many times larger output file # should explicitly switch the debug mode on - # otherwise we let dllwrap/ld strip the output file + # otherwise we let ld strip the output file # (On my machine: 10KiB < stripped_file < ??100KiB # unstripped_file = stripped_file + XXX KiB # ( XXX=254 for a typical python extension)) - if not debug: + if not debug and not hasattr(sys, 'gettotalrefcount'): extra_preargs.append("-s") UnixCCompiler.link(self, target_desc, objects, output_filename, @@ -253,11 +249,16 @@ def object_filenames(self, source_filenames, strip_dir=0, output_dir=''): output_dir = '' obj_names = [] for src_name in source_filenames: - # use normcase to make sure '.rc' is really '.rc' and not '.RC' - base, ext = os.path.splitext(os.path.normcase(src_name)) - if ext not in (self.src_extensions + ['.rc','.res']): + base, ext = os.path.splitext(src_name) + # use 'normcase' only for resource suffixes + ext_normcase = os.path.normcase(ext) + if ext_normcase in ['.rc', '.res', '.mc']: + ext = ext_normcase + if ext not in (self.src_extensions + ['.rc', '.res', '.mc']): raise UnknownFileError("unknown file type '%s' (from '%s')" % \ (ext, src_name)) + base = os.path.splitdrive(base)[1] # Chop off the drive + base = base[os.path.isabs(base):] # If abs, chop off leading / if strip_dir: base = os.path.basename (base) if ext in ('.res', '.rc'): @@ -279,31 +280,18 @@ def __init__(self, verbose=0, dry_run=0, force=0): CygwinCCompiler.__init__ (self, verbose, dry_run, force) - # ld_version >= "2.13" support -shared so use it instead of - # -mdll -static - if self.ld_version >= "2.13": - shared_option = "-shared" - else: - shared_option = "-mdll -static" - - # A real mingw32 doesn't need to specify a different entry point, - # but cygwin 2.91.57 in no-cygwin-mode needs it. - if self.gcc_version <= "2.91.57": - entry_point = '--entry _DllMain@12' - else: - entry_point = '' + shared_option = "-shared" - if is_cygwingcc(): + if is_cygwincc(self.cc): raise CCompilerError( 'Cygwin gcc cannot be used with --compiler=mingw32') - self.set_executables(compiler='gcc -O -Wall', - compiler_so='gcc -mdll -O -Wall', - compiler_cxx='g++ -O -Wall', - linker_exe='gcc', - linker_so='%s %s %s' - % (self.linker_dll, shared_option, - entry_point)) + self.set_executables(compiler='%s -O2 -Wall' % self.cc, + compiler_so='%s -mdll -O2 -Wall' % self.cc, + compiler_cxx='%s -O2 -Wall' % self.cxx, + linker_exe='%s' % self.cc, + linker_so='%s %s' + % (self.linker_dll, shared_option)) # Maybe we should also append -mthreads, but then the finished # dlls need another dll (mingwm10.dll see Mingw32 docs) # (-mthreads: Support thread-safe exception handling on `Mingw32') @@ -313,7 +301,7 @@ def __init__(self, verbose=0, dry_run=0, force=0): # Include the appropriate MSVC runtime library if Python was built # with MSVC 7.0 or later. - self.dll_libraries = get_msvcr() + self.dll_libraries = get_msvcr() or [] # Because these compilers aren't configured in Python's pyconfig.h file by # default, we should at least warn the user if he is using an unmodified @@ -351,6 +339,10 @@ def check_config_h(): if "GCC" in sys.version: return CONFIG_H_OK, "sys.version mentions 'GCC'" + # Clang would also work + if "Clang" in sys.version: + return CONFIG_H_OK, "sys.version mentions 'Clang'" + # let's see if __GNUC__ is mentioned in python.h fn = sysconfig.get_config_h_filename() try: @@ -366,38 +358,8 @@ def check_config_h(): return (CONFIG_H_UNCERTAIN, "couldn't read '%s': %s" % (fn, exc.strerror)) -RE_VERSION = re.compile(br'(\d+\.\d+(\.\d+)*)') - -def _find_exe_version(cmd): - """Find the version of an executable by running `cmd` in the shell. - - If the command is not found, or the output does not match - `RE_VERSION`, returns None. - """ - executable = cmd.split()[0] - if find_executable(executable) is None: - return None - out = Popen(cmd, shell=True, stdout=PIPE).stdout - try: - out_string = out.read() - finally: - out.close() - result = RE_VERSION.search(out_string) - if result is None: - return None - # LooseVersion works with strings - # so we need to decode our bytes - return LooseVersion(result.group(1).decode()) - -def get_versions(): - """ Try to find out the versions of gcc, ld and dllwrap. - - If not possible it returns None for it. - """ - commands = ['gcc -dumpversion', 'ld -v', 'dllwrap --version'] - return tuple([_find_exe_version(cmd) for cmd in commands]) -def is_cygwingcc(): - '''Try to determine if the gcc that would be used is from cygwin.''' - out_string = check_output(['gcc', '-dumpmachine']) +def is_cygwincc(cc): + '''Try to determine if the compiler that would be used is from cygwin.''' + out_string = check_output(shlex.split(cc) + ['-dumpmachine']) return out_string.strip().endswith(b'cygwin') diff --git a/Lib/distutils/msvc9compiler.py b/Lib/distutils/msvc9compiler.py index a7976fbe3ed924..c3416797d5f55a 100644 --- a/Lib/distutils/msvc9compiler.py +++ b/Lib/distutils/msvc9compiler.py @@ -291,8 +291,6 @@ def query_vcvarsall(version, arch="x86"): # More globals VERSION = get_build_version() -if VERSION < 8.0: - raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) # MACROS = MacroExpander(VERSION) class MSVCCompiler(CCompiler) : @@ -327,6 +325,8 @@ class MSVCCompiler(CCompiler) : def __init__(self, verbose=0, dry_run=0, force=0): CCompiler.__init__ (self, verbose, dry_run, force) + if VERSION < 8.0: + raise DistutilsPlatformError("VC %0.1f is not supported by this module" % VERSION) self.__version = VERSION self.__root = r"Software\Microsoft\VisualStudio" # self.__macros = MACROS diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 03b85584190db9..024c15c8a82c8d 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -64,8 +64,23 @@ def parse_config_h(fp, g=None): _python_build = partial(is_python_build, check_home=True) _init_posix = partial(sysconfig_init_posix, _config_vars) -_init_nt = partial(_init_non_posix, _config_vars) +def _posix_build(): + # GCC[mingw*] use posix build system + # Check for cross builds explicitly + host_platform = os.environ.get("_PYTHON_HOST_PLATFORM") + if host_platform: + if host_platform.startswith('mingw'): + return True + return os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) +posix_build = _posix_build() + + +def _init_nt(): + if posix_build: + return _init_posix(_config_vars) + return _init_non_posix(_config_vars) # Similar function is also implemented in sysconfig as _parse_makefile # but without the parsing capabilities of distutils.text_file.TextFile. @@ -196,7 +211,23 @@ def customize_compiler(compiler): Mainly needed on Unix, so we can plug in the information that varies across Unices and is stored in Python's Makefile. """ - if compiler.compiler_type == "unix": + global _config_vars + if compiler.compiler_type in ["cygwin", "mingw32"]: + # Note that cygwin use posix build and 'unix' compiler. + # If build is not based on posix then we must predefine + # some environment variables corresponding to posix + # build rules and defaults. + if not 'GCC' in sys.version: + _config_vars['CC'] = "gcc" + _config_vars['CXX'] = "g++" + _config_vars['OPT'] = "-fwrapv -O3 -Wall -Wstrict-prototypes" + _config_vars['CFLAGS'] = "" + _config_vars['CCSHARED'] = "" + _config_vars['LDSHARED'] = "gcc -shared -Wl,--enable-auto-image-base" + _config_vars['AR'] = "ar" + _config_vars['ARFLAGS'] = "rc" + + if compiler.compiler_type in ["unix", "cygwin", "mingw32"]: if sys.platform == "darwin": # Perform first-time customization of compiler-related # config vars on OS X now that we know we need a compiler. @@ -274,7 +305,7 @@ def get_python_inc(plat_specific=0, prefix=None): """ if prefix is None: prefix = plat_specific and BASE_EXEC_PREFIX or BASE_PREFIX - if os.name == "posix": + if posix_build: if python_build: # Assume the executable is in the build directory. The # pyconfig.h file should be in the same directory. Since @@ -321,7 +352,7 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None): else: prefix = plat_specific and EXEC_PREFIX or PREFIX - if os.name == "posix": + if posix_build: if plat_specific or standard_lib: # Platform-specific modules (any module from a non-pure-Python # module distribution) or standard Python library modules. diff --git a/Lib/distutils/tests/test_cygwinccompiler.py b/Lib/distutils/tests/test_cygwinccompiler.py index 0912ffd15c8ee9..a93c174758b33d 100644 --- a/Lib/distutils/tests/test_cygwinccompiler.py +++ b/Lib/distutils/tests/test_cygwinccompiler.py @@ -8,7 +8,7 @@ from distutils import cygwinccompiler from distutils.cygwinccompiler import (check_config_h, CONFIG_H_OK, CONFIG_H_NOTOK, - CONFIG_H_UNCERTAIN, get_versions, + CONFIG_H_UNCERTAIN, get_msvcr) from distutils.tests import support @@ -81,40 +81,6 @@ def test_check_config_h(self): self.write_file(self.python_h, 'xxx __GNUC__ xxx') self.assertEqual(check_config_h()[0], CONFIG_H_OK) - def test_get_versions(self): - - # get_versions calls distutils.spawn.find_executable on - # 'gcc', 'ld' and 'dllwrap' - self.assertEqual(get_versions(), (None, None, None)) - - # Let's fake we have 'gcc' and it returns '3.4.5' - self._exes['gcc'] = b'gcc (GCC) 3.4.5 (mingw special)\nFSF' - res = get_versions() - self.assertEqual(str(res[0]), '3.4.5') - - # and let's see what happens when the version - # doesn't match the regular expression - # (\d+\.\d+(\.\d+)*) - self._exes['gcc'] = b'very strange output' - res = get_versions() - self.assertEqual(res[0], None) - - # same thing for ld - self._exes['ld'] = b'GNU ld version 2.17.50 20060824' - res = get_versions() - self.assertEqual(str(res[1]), '2.17.50') - self._exes['ld'] = b'@(#)PROGRAM:ld PROJECT:ld64-77' - res = get_versions() - self.assertEqual(res[1], None) - - # and dllwrap - self._exes['dllwrap'] = b'GNU dllwrap 2.17.50 20060824\nFSF' - res = get_versions() - self.assertEqual(str(res[2]), '2.17.50') - self._exes['dllwrap'] = b'Cheese Wrap' - res = get_versions() - self.assertEqual(res[2], None) - def test_get_msvcr(self): # none diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index d00c48981eb6d6..76a7d824760d60 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -249,9 +249,13 @@ def runtime_library_dir_option(self, dir): # -Wl whenever gcc was used in the past it is probably # safest to keep doing so. if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH + # GNU ELF ld needs an extra option to get a RUNPATH # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir + if sys.platform in ["win32", "cygwin"] or \ + "mingw" in compiler: + return [] + else: + return "-Wl,--enable-new-dtags,-R" + dir else: return "-Wl,-R" + dir else: diff --git a/Lib/distutils/util.py b/Lib/distutils/util.py index 2ce5c5b64d62fa..81626b394612c2 100644 --- a/Lib/distutils/util.py +++ b/Lib/distutils/util.py @@ -37,6 +37,22 @@ def get_host_platform(): """ if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): @@ -130,6 +146,13 @@ def convert_path (pathname): paths.remove('.') if not paths: return os.curdir + # On Windows, if paths is ['C:','folder','subfolder'] then + # os.path.join(*paths) will return 'C:folder\subfolder' which + # is thus relative to the CWD on that drive. So we work around + # this by adding a \ to path[0] + if (len(paths) > 0 and paths[0].endswith(':') and + sys.platform == "win32" and sys.version.find("GCC") >= 0): + paths[0] += '\\' return os.path.join(*paths) # convert_path () @@ -140,6 +163,10 @@ def change_root (new_root, pathname): relative, this is equivalent to "os.path.join(new_root,pathname)". Otherwise, it requires making 'pathname' relative and then joining the two, which is tricky on DOS/Windows and Mac OS. + + If on Windows or OS/2 and both new_root and pathname are on different + drives, raises DistutilsChangeRootError as this is nonsensical, + otherwise use drive which can be in either of new_root or pathname. """ if os.name == 'posix': if not os.path.isabs(pathname): @@ -149,9 +176,20 @@ def change_root (new_root, pathname): elif os.name == 'nt': (drive, path) = os.path.splitdrive(pathname) - if path[0] == '\\': + if path[0] == os.sep: path = path[1:] - return os.path.join(new_root, path) + (drive_r, path_r) = os.path.splitdrive(new_root) + if path_r and path_r[0] == os.sep: + path_r = path_r[1:] + drive_used = '' + if len(drive) == 2 and len(drive_r) == 2 and drive != drive_r: + raise DistutilsChangeRootError("root and pathname not on same drive (%s, %s)" + % (drive_r,drive)) + elif len(drive_r) == 2: + drive_used = drive_r+os.sep + elif len(drive) == 2: + drive_used = drive+os.sep + return os.path.join(drive_used+path_r, path) else: raise DistutilsPlatformError("nothing known about platform '%s'" % os.name) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index f603a89f7f5a98..7693d051d4285d 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -42,6 +42,10 @@ path_separators = ['\\', '/'] else: path_separators = ['/'] + +if _os.environ.get('MSYSTEM', ''): + path_separators = path_separators[::-1] + # Assumption made in _path_join() assert all(len(sep) == 1 for sep in path_separators) path_sep = path_separators[0] diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 0246419485da0d..7d5c3a5d7f9b08 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -11,9 +11,7 @@ curdir = '.' pardir = '..' extsep = '.' -sep = '\\' pathsep = ';' -altsep = '/' defpath = '.;C:\\bin' devnull = 'nul' @@ -23,6 +21,14 @@ import genericpath from genericpath import * +if sys.platform == "win32" and os.environ.get("MSYSTEM", ""): + sep = '/' + altsep = '\\' +else: + sep = '\\' + altsep = '/' +bsep = str.encode(sep) +baltsep = str.encode(altsep) __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -34,9 +40,33 @@ def _get_bothseps(path): if isinstance(path, bytes): - return b'\\/' + return bsep+baltsep + else: + return sep+altsep + +def _get_sep(path): + if isinstance(path, bytes): + return bsep + else: + return sep + +def _get_altsep(path): + if isinstance(path, bytes): + return baltsep + else: + return altsep + +def _get_colon(path): + if isinstance(path, bytes): + return b':' + else: + return ':' + +def _get_unc_prefix(path): + if isinstance(path, bytes): + return b'\\\\?\\UNC\\' else: - return '\\/' + return '\\\\?\\UNC\\' # Normalize the case of a pathname and map slashes to backslashes. # Other normalizations (such as optimizing '../' away) are not done @@ -58,14 +88,14 @@ def normcase(s): return s if isinstance(s, bytes): encoding = sys.getfilesystemencoding() - s = s.decode(encoding, 'surrogateescape').replace('/', '\\') + s = s.decode(encoding, 'surrogateescape').replace(altsep, sep) s = _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, s) return s.encode(encoding, 'surrogateescape') else: return _LCMapStringEx(_LOCALE_NAME_INVARIANT, _LCMAP_LOWERCASE, - s.replace('/', '\\')) + s.replace(altsep, sep)) except ImportError: def normcase(s): """Normalize case of pathname. @@ -74,8 +104,8 @@ def normcase(s): """ s = os.fspath(s) if isinstance(s, bytes): - return os.fsencode(os.fsdecode(s).replace('/', '\\').lower()) - return s.replace('/', '\\').lower() + return os.fsencode(os.fsdecode(s).replace(altsep, sep).lower()) + return s.replace(altsep, sep).lower() # Return whether a path is absolute. @@ -87,14 +117,9 @@ def normcase(s): def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) - if isinstance(s, bytes): - sep = b'\\' - altsep = b'/' - colon_sep = b':\\' - else: - sep = '\\' - altsep = '/' - colon_sep = ':\\' + sep = _get_sep(s) + altsep = _get_altsep(s) + colon_sep = _get_colon(s) + sep s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. # LEGACY BUG: isabs("/x") should be false since the path has no drive. @@ -106,14 +131,9 @@ def isabs(s): # Join two (or more) paths. def join(path, *paths): path = os.fspath(path) - if isinstance(path, bytes): - sep = b'\\' - seps = b'\\/' - colon = b':' - else: - sep = '\\' - seps = '\\/' - colon = ':' + sep = _get_sep(path) + seps = _get_bothseps(path) + colon = _get_colon(path) try: if not paths: path[:0] + sep #23780: Ensure compatible data type even if p is null. @@ -172,16 +192,10 @@ def splitdrive(p): """ p = os.fspath(p) if len(p) >= 2: - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' + sep = _get_sep(p) + altsep = _get_altsep(p) + colon = _get_colon(p) + unc_prefix = _get_unc_prefix(p) normp = p.replace(altsep, sep) if normp[0:2] == sep * 2: # UNC drives, e.g. \\server\share or \\?\UNC\server\share @@ -231,9 +245,9 @@ def split(p): def splitext(p): p = os.fspath(p) if isinstance(p, bytes): - return genericpath._splitext(p, b'\\', b'/', b'.') + return genericpath._splitext(p, bsep, baltsep, b'.') else: - return genericpath._splitext(p, '\\', '/', '.') + return genericpath._splitext(p, sep, altsep, '.') splitext.__doc__ = genericpath._splitext.__doc__ @@ -334,7 +348,7 @@ def expanduser(path): if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif not 'HOMEPATH' in os.environ: - return path + return os.path.normpath(path) else: try: drive = os.environ['HOMEDRIVE'] @@ -361,7 +375,7 @@ def expanduser(path): if isinstance(path, bytes): userhome = os.fsencode(userhome) - return userhome + path[i:] + return os.path.normpath(userhome) + path[i:] # Expand paths containing shell variable substitutions. @@ -496,14 +510,12 @@ def expandvars(path): def normpath(path): """Normalize path, eliminating double slashes, etc.""" path = os.fspath(path) + sep = _get_sep(path) + altsep = _get_altsep(path) if isinstance(path, bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' pardir = b'..' else: - sep = '\\' - altsep = '/' curdir = '.' pardir = '..' path = path.replace(altsep, sep) @@ -570,7 +582,7 @@ def _abspath_fallback(path): def abspath(path): """Return the absolute version of a path.""" try: - return _getfullpathname(normpath(path)) + return normpath(_getfullpathname(normpath(path))) except (OSError, ValueError): return _abspath_fallback(path) @@ -731,6 +743,7 @@ def realpath(path, *, strict=False): # strip the prefix anyway. if ex.winerror == initial_winerror: path = spath + path = normpath(path) return path @@ -741,12 +754,11 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + sep = _get_sep(path) if isinstance(path, bytes): - sep = b'\\' curdir = b'.' pardir = b'..' else: - sep = '\\' curdir = '.' pardir = '..' @@ -801,13 +813,11 @@ def commonpath(paths): raise ValueError('commonpath() arg is an empty sequence') paths = tuple(map(os.fspath, paths)) + sep = _get_sep(paths[0]) + altsep = _get_altsep(paths[0]) if isinstance(paths[0], bytes): - sep = b'\\' - altsep = b'/' curdir = b'.' else: - sep = '\\' - altsep = '/' curdir = '.' try: diff --git a/Lib/pathlib.py b/Lib/pathlib.py index ecb1e8a40d8cf0..eec8a9e9468df1 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -115,6 +115,8 @@ class _WindowsFlavour(_Flavour): sep = '\\' altsep = '/' + if os.environ.get('MSYSTEM', ''): + sep, altsep = altsep, sep has_drv = True pathmod = ntpath diff --git a/Lib/site.py b/Lib/site.py index 69670d9d7f2230..41b9cf1a8d2bbf 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -88,6 +88,12 @@ USER_BASE = None +# Same as defined in Lib/sysconfig.py +# redeclared since sysconfig is large for site. +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + def _trace(message): if sys.flags.verbose: print(message, file=sys.stderr) @@ -273,7 +279,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -283,14 +289,36 @@ def joinuser(*args): return joinuser("~", ".local") +# Copy of sysconfig.get_platform() but only for MinGW +def _get_platform(): + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' + return sys.platform # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info if os.name == 'nt': - ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\Python{ver_nodot}\\site-packages' + if not _POSIX_BUILD: + ver_nodot = sys.winver.replace('.', '') + return f'{userbase}\\Python{ver_nodot}\\site-packages' + return f'{userbase}/lib/python{version[0]}.{version[1]}-{_get_platform()}/site-packages' if sys.platform == 'darwin' and sys._framework: return f'{userbase}/lib/python/site-packages' @@ -361,7 +389,7 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) - if os.sep == '/': + if _POSIX_BUILD: libdirs = [sys.platlibdir] if sys.platlibdir != "lib": libdirs.append("lib") @@ -392,7 +420,7 @@ def setquit(): The repr of each object contains a hint at how it works. """ - if os.sep == '\\': + if sys.platform == 'win32': eof = 'Ctrl-Z plus Return' else: eof = 'Ctrl-D (i.e. EOF)' diff --git a/Lib/ssl.py b/Lib/ssl.py index 48d229f810af28..5d331c15ba4ecc 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -254,7 +254,7 @@ class _TLSMessageType: CHANGE_CIPHER_SPEC = 0x0101 -if sys.platform == "win32": +if sys.platform == "win32" and sys.version.find("GCC") == -1: from _ssl import enum_certificates, enum_crls from socket import socket, SOCK_STREAM, create_connection @@ -591,7 +591,7 @@ def _load_windows_store_certs(self, storename, purpose): def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): raise TypeError(purpose) - if sys.platform == "win32": + if sys.platform == "win32" and sys.version.find("GCC") == -1: for storename in self._windows_cert_stores: self._load_windows_store_certs(storename, purpose) self.set_default_verify_paths() diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index ebe37118274222..8035d2e273e33c 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -2,6 +2,7 @@ import os import sys +import textwrap from os.path import pardir, realpath __all__ = [ @@ -47,13 +48,13 @@ 'data': '{base}', }, 'nt': { - 'stdlib': '{installed_base}/Lib', - 'platstdlib': '{base}/Lib', - 'purelib': '{base}/Lib/site-packages', - 'platlib': '{base}/Lib/site-packages', - 'include': '{installed_base}/Include', - 'platinclude': '{installed_base}/Include', - 'scripts': '{base}/Scripts', + 'stdlib': '{installed_base}/lib/python{py_version_short}', + 'platstdlib': '{base}/lib/python{py_version_short}', + 'purelib': '{base}/lib/python{py_version_short}/site-packages', + 'platlib': '{base}/lib/python{py_version_short}/site-packages', + 'include': '{installed_base}/include/python{py_version_short}', + 'platinclude': '{installed_base}/include/python{py_version_short}', + 'scripts': '{base}/bin', 'data': '{base}', }, # Downstream distributors can overwrite the default install scheme. @@ -97,13 +98,18 @@ }, } +# GCC[mingw*] use posix build system +_POSIX_BUILD = os.name == 'posix' or \ + (os.name == "nt" and 'GCC' in sys.version) + # For the OS-native venv scheme, we essentially provide an alias: -if os.name == 'nt': +if os.name == 'nt' and not _POSIX_BUILD: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['nt_venv'] else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] + # NOTE: site.py has copy of this function. # Sync it when modify this function. def _getuserbase(): @@ -118,7 +124,7 @@ def _getuserbase(): def joinuser(*args): return os.path.expanduser(os.path.join(*args)) - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: base = os.environ.get("APPDATA") or "~" return joinuser(base, "Python") @@ -134,20 +140,20 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/lib/python{py_version_short_plat}', + 'platstdlib': '{userbase}/lib/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', + 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short_plat}', + 'purelib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'platlib': '{userbase}/lib/python{py_version_short_plat}/site-packages', + 'include': '{userbase}/include/python{py_version_short_plat}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -277,7 +283,7 @@ def _expand_vars(scheme, vars): def _get_preferred_schemes(): - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: return { 'prefix': 'nt', 'home': 'posix_home', @@ -435,6 +441,14 @@ def _parse_makefile(filename, vars=None, keep_unresolved=True): if isinstance(v, str): done[k] = v.strip() + # any keys that have one with the same name suffixed with _b2h + # need to be replaced with the value of the _b2h key. + # This converts from MSYS*/Cygwin paths to Windows paths. + for k, v in dict(done).items(): + if isinstance(k, str): + if k.endswith("_b2h"): + done[k[:-4]]=v + # save the results in the global dictionary vars.update(done) return vars @@ -514,11 +528,30 @@ def _generate_posix_vars(): os.makedirs(pybuilddir, exist_ok=True) destfile = os.path.join(pybuilddir, name + '.py') + replacement = """ + keys_to_replace = [ + 'BINDIR', 'BINLIBDEST', 'CONFINCLUDEDIR', + 'CONFINCLUDEPY', 'DESTDIRS', 'DESTLIB', 'DESTSHARED', + 'INCLDIRSTOMAKE', 'INCLUDEDIR', 'INCLUDEPY', + 'LIBDEST', 'LIBDIR', 'LIBPC', 'LIBPL', 'MACHDESTLIB', + 'MANDIR', 'SCRIPTDIR', 'datarootdir', 'exec_prefix', + 'TZPATH', + ] + + prefix = build_time_vars['BINDIR'][:-4] + + for key in keys_to_replace: + value = build_time_vars[key] + build_time_vars[key] = value.replace(prefix, sys.prefix) + """ + with open(destfile, 'w', encoding='utf8') as f: + f.write('import sys\n') f.write('# system configuration generated and used by' ' the sysconfig module\n') f.write('build_time_vars = ') pprint.pprint(vars, stream=f) + f.write('\n%s' % textwrap.dedent(replacement)) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: @@ -541,7 +574,7 @@ def _init_non_posix(vars): vars['INCLUDEPY'] = get_path('include') vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] vars['EXE'] = '.exe' - vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT + vars['VERSION'] = _PY_VERSION_SHORT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) vars['TZPATH'] = '' @@ -587,7 +620,7 @@ def parse_config_h(fp, vars=None): def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: - if os.name == "nt": + if os.name == "nt" and not _POSIX_BUILD: inc_dir = os.path.join(_PROJECT_BASE, "PC") else: inc_dir = _PROJECT_BASE @@ -662,11 +695,15 @@ def get_config_vars(*args): _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: _CONFIG_VARS['py_version_nodot_plat'] = '' + if os.name == 'nt' and _POSIX_BUILD: + _CONFIG_VARS['py_version_short_plat'] = f'{_PY_VERSION_SHORT}-{get_platform()}' + else: + _CONFIG_VARS['py_version_short_plat'] = _PY_VERSION_SHORT - if os.name == 'nt': + if os.name == 'nt' and not _POSIX_BUILD: _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': + if _POSIX_BUILD: _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the @@ -676,7 +713,7 @@ def get_config_vars(*args): # Always convert srcdir to an absolute path srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': + if _POSIX_BUILD: if _PYTHON_BUILD: # If srcdir is a relative path (typically '.' or '..') # then it should be interpreted relative to the directory @@ -714,7 +751,7 @@ def get_config_var(name): """ return get_config_vars().get(name) - +# make sure to change site._get_platform() while changing this function def get_platform(): """Return a string that identifies the current platform. @@ -737,6 +774,22 @@ def get_platform(): """ if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if 'amd64' in sys.version.lower(): return 'win-amd64' if '(arm)' in sys.version.lower(): diff --git a/Lib/test/__main__.py b/Lib/test/__main__.py index 19a6b2b8904526..7641b32c2d4d1b 100644 --- a/Lib/test/__main__.py +++ b/Lib/test/__main__.py @@ -1,2 +1,10 @@ +import os +import sys + from test.libregrtest import main + +if sys.platform == "win32": + # Enable DLL loading from PATH. + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + main() diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 53ba1ad6f5911f..3b6f4ab197b91b 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1105,7 +1105,7 @@ def test_from_format(self): if os.name == 'nt': # Windows (MSCRT) - ptr_format = '0x%0{}X'.format(2 * sizeof_ptr) + ptr_format = '0x%0{}x'.format(2 * sizeof_ptr) def ptr_formatter(ptr): return (ptr_format % ptr) else: diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 4558560e1a992f..11d5001bc63793 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -1372,6 +1372,8 @@ def test_shutdown_deadlock_pickle(self): executor_manager.join() def test_crash_big_data(self): + # https://github.com/python/cpython/issues/107219 + return # Test that there is a clean exception instad of a deadlock when a # child process crashes while some data is being written into the # queue. diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index 8d5b4265f269ee..82ad71ddd1ed00 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -838,6 +838,7 @@ def test_symlink_buildpath_macos(self): ENV_PYTHONHOME="", ENV_PYTHONEXECUTABLE="", ENV___PYVENV_LAUNCHER__="", + ENV_MSYSTEM="", argv0="", py_setpath="", real_executable="", @@ -877,6 +878,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "nt" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "DLLs" self["PYWINVER"] = "9.8-XY" self["VPATH"] = r"..\.." @@ -912,6 +914,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return ntpath.normpath(path) + def abspath(self, path): if self.isabs(path): return path @@ -1053,6 +1058,7 @@ def __init__(self, *a, argv0=None, config=None, **kw): self.update(DEFAULT_NAMESPACE) self["config"] = DEFAULT_CONFIG.copy() self["os_name"] = "posix" + self["is_mingw"] = 0 self["PLATLIBDIR"] = "lib" self["WITH_NEXT_FRAMEWORK"] = 0 super().__init__(*a, **kw) @@ -1089,6 +1095,9 @@ def __missing__(self, key): except AttributeError: raise KeyError(key) from None + def normpath(self, path): + return path + def abspath(self, path): if self.isabs(path): return path diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index cdd1bea754a020..cfd8a101dcc1c1 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -442,10 +442,10 @@ def test_undecodable_filename(self): def test_undecodable_parameter(self): # sanity check using a valid parameter response = self.request(self.base_url + '/?x=123').read() - self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1')) + self.assertRegex(response, rf'listing for {self.base_url}/\?x=123'.encode('latin1')) # now the bogus encoding response = self.request(self.base_url + '/?x=%bb').read() - self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + self.assertRegex(response, rf'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index b7dfe865a03a97..58a8f5ddf090f9 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -24,6 +24,23 @@ def get_platform(): 'x64' : 'win-amd64', 'arm' : 'win-arm32', } + if os.name == 'nt': + if 'gcc' in sys.version.lower(): + if 'ucrt' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_ucrt' + return 'mingw_i686_ucrt' + if 'clang' in sys.version.lower(): + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64_clang' + if 'arm64' in sys.version.lower(): + return 'mingw_aarch64' + if 'arm' in sys.version.lower(): + return 'mingw_armv7' + return 'mingw_i686_clang' + if 'amd64' in sys.version.lower(): + return 'mingw_x86_64' + return 'mingw_i686' if ('VSCMD_ARG_TGT_ARCH' in os.environ and os.environ['VSCMD_ARG_TGT_ARCH'] in TARGET_TO_PLAT): return TARGET_TO_PLAT[os.environ['VSCMD_ARG_TGT_ARCH']] diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index d96371d24269f7..a75a664679d797 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -17,7 +17,7 @@ from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes, _main, _POSIX_BUILD) import _osx_support @@ -180,7 +180,7 @@ def test_nt_venv_scheme(self): self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) def test_venv_scheme(self): - if sys.platform == 'win32': + if not _POSIX_BUILD and sys.platform == 'win32': self.assertEqual( sysconfig.get_path('scripts', scheme='venv'), sysconfig.get_path('scripts', scheme='nt_venv') @@ -371,6 +371,10 @@ def test_user_similar(self): if HAS_USER_BASE: user_path = get_path(name, 'posix_user') expected = os.path.normpath(global_path.replace(base, user, 1)) + if os.name == 'nt' and _POSIX_BUILD: + expected = expected.replace( + f'python{sysconfig.get_python_version()}', + f'python{sysconfig.get_python_version()}-{get_platform()}') # bpo-44860: platlib of posix_user doesn't use sys.platlibdir, # whereas posix_prefix does. if name == 'platlib': diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6bce3081088200..1cd57d05f696c1 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,7 +11,7 @@ import sys import sysconfig import types - +from sysconfig import _POSIX_BUILD CORE_VENV_DEPS = ('pip', 'setuptools') logger = logging.getLogger(__name__) @@ -301,7 +301,7 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) else: - if self.symlinks: + if self.symlinks and not _POSIX_BUILD: # For symlinking, we need a complete copy of the root directory # If symlinks fail, you'll get unnecessary copies of files, but # we assume that if you've opted into symlinks on Windows then @@ -325,6 +325,12 @@ def setup_python(self, context): if os.path.lexists(src): copier(src, os.path.join(binpath, suffix)) + if _POSIX_BUILD: + # copy from python/pythonw so the venvlauncher magic in symlink_or_copy triggers + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python3.exe')) + copier(os.path.join(dirname, 'python.exe'), os.path.join(binpath, 'python%d.%d.exe' % sys.version_info[:2])) + copier(os.path.join(dirname, 'pythonw.exe'), os.path.join(binpath, 'python3w.exe')) + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): @@ -349,6 +355,7 @@ def _call_new_python(self, context, *py_args, **kwargs): env['VIRTUAL_ENV'] = context.env_dir env.pop('PYTHONHOME', None) env.pop('PYTHONPATH', None) + env.pop("MSYSTEM", None) kwargs['cwd'] = context.env_dir kwargs['executable'] = context.env_exec_cmd subprocess.check_output(args, **kwargs) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fbc2b8801da04..9c3d58dfd61ed9 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -38,7 +38,7 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" +VIRTUAL_ENV=$(cygpath "__VENV_DIR__") export VIRTUAL_ENV _OLD_VIRTUAL_PATH="$PATH" diff --git a/Makefile.pre.in b/Makefile.pre.in index 884bea1841c232..a2ccbf134b6a54 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -39,6 +39,7 @@ CXX= @CXX@ MAINCC= @MAINCC@ LINKCC= @LINKCC@ AR= @AR@ +WINDRES= @WINDRES@ READELF= @READELF@ SOABI= @SOABI@ LDVERSION= @LDVERSION@ @@ -123,6 +124,7 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ +RCFLAGS=@RCFLAGS@ # Machine-dependent subdirectories @@ -141,6 +143,12 @@ exec_prefix= @exec_prefix@ # Install prefix for data files datarootdir= @datarootdir@ +# Locations needed for semi-native fixup of sysconfig. +srcdir_b2h= @srcdir_b2h@ +abs_srcdir_b2h= @abs_srcdir_b2h@ +abs_builddir_b2h= @abs_builddir_b2h@ +prefix_b2h= @prefix_b2h@ + # Expanded directories BINDIR= @bindir@ LIBDIR= @libdir@ @@ -158,10 +166,12 @@ BINLIBDEST= @BINLIBDEST@ LIBDEST= $(SCRIPTDIR)/python$(VERSION) INCLUDEPY= $(INCLUDEDIR)/python$(LDVERSION) CONFINCLUDEPY= $(CONFINCLUDEDIR)/python$(LDVERSION) +VENVLAUNCHERDIR= $(BINLIBDEST)/venv/scripts/nt # Symbols used for using shared libraries SHLIB_SUFFIX= @SHLIB_SUFFIX@ EXT_SUFFIX= @EXT_SUFFIX@ +PYD_PLATFORM_TAG = @PYD_PLATFORM_TAG@ LDSHARED= @LDSHARED@ $(PY_LDFLAGS) BLDSHARED= @BLDSHARED@ $(PY_CORE_LDFLAGS) LDCXXSHARED= @LDCXXSHARED@ @@ -268,6 +278,8 @@ LIBRARY_DEPS= @LIBRARY_DEPS@ LINK_PYTHON_DEPS=@LINK_PYTHON_DEPS@ PY_ENABLE_SHARED= @PY_ENABLE_SHARED@ STATIC_LIBPYTHON= @STATIC_LIBPYTHON@ +ABI3DLLLIBRARY= libpython3.dll +ABI3LDLIBRARY= libpython3.dll.a LIBS= @LIBS@ @@ -284,6 +296,9 @@ LIBOBJS= @LIBOBJS@ PYTHON= python$(EXE) BUILDPYTHON= python$(BUILDEXE) +BUILDPYTHONW= pythonw$(BUILDEXE) +BUILDVENVLAUNCHER= venvlauncher$(BUILDEXE) +BUILDVENVWLAUNCHER= venvwlauncher$(BUILDEXE) HOSTRUNNER= @HOSTRUNNER@ @@ -344,6 +359,7 @@ IO_OBJS= \ ########################################################################## LIBFFI_INCLUDEDIR= @LIBFFI_INCLUDEDIR@ +NCURSESW_INCLUDEDIR= @NCURSESW_INCLUDEDIR@ ########################################################################## # Parser @@ -390,7 +406,7 @@ PYTHON_OBJS= \ Python/dynamic_annotations.o \ Python/errors.o \ Python/frame.o \ - Python/frozenmain.o \ + @PYTHON_OBJS_FROZENMAIN@ \ Python/future.o \ Python/getargs.o \ Python/getcompiler.o \ @@ -402,6 +418,7 @@ PYTHON_OBJS= \ Python/import.o \ Python/importdl.o \ Python/initconfig.o \ + Python/iscygpty.o \ Python/marshal.o \ Python/modsupport.o \ Python/mysnprintf.o \ @@ -584,8 +601,8 @@ LIBEXPAT_HEADERS= \ # Default target all: @DEF_MAKE_ALL_RULE@ -build_all: check-clean-src $(BUILDPYTHON) platform oldsharedmods sharedmods \ - gdbhooks Programs/_testembed python-config +build_all: check-clean-src $(BUILDPYTHON) $(BUILDPYTHONW) $(BUILDVENVLAUNCHER) $(BUILDVENVWLAUNCHER) platform oldsharedmods sharedmods \ + gdbhooks Programs/_testembed python-config $(ABI3DLLLIBRARY) $(ABI3LDLIBRARY) build_wasm: check-clean-src $(BUILDPYTHON) platform oldsharedmods python-config # Check that the source is clean when building out of source. @@ -700,9 +717,36 @@ coverage-report: regen-token regen-frozen clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c $(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir) +python_exe.o: $(srcdir)/PC/python_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_exe.rc $@ + +pythonw_exe.o: $(srcdir)/PC/pythonw_exe.rc + $(WINDRES) $(RCFLAGS) -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pythonw_exe.rc $@ + +python_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + +python3dll_nt.o: $(srcdir)/PC/python_nt.rc + $(WINDRES) $(RCFLAGS) -DORIGINAL_FILENAME=\\\"$(ABI3DLLLIBRARY)\\\" -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/python_nt.rc $@ + +venvlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPY_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + +venvwlauncher.o: $(srcdir)/PC/pylauncher.rc + $(WINDRES) $(RCFLAGS) -DPYW_ICON -I$(srcdir)/Include -I$(srcdir)/PC -I. $(srcdir)/PC/pylauncher.rc $@ + +$(BUILDPYTHONW): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) pythonw_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -mwindows -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) pythonw_exe.o + # Build the interpreter -$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) - $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) +$(BUILDPYTHON): Programs/python.o $(LINK_PYTHON_DEPS) python_exe.o + $(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -municode -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS) python_exe.o + +$(BUILDVENVLAUNCHER): $(BUILDPYTHON) venvlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_CONSOLE -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"python$(LDVERSION)$(EXE)\"" -municode -static -static-libgcc -static-libstdc++ venvlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion + +$(BUILDVENVWLAUNCHER): $(BUILDPYTHONW) venvwlauncher.o $(srcdir)/PC/launcher.c + $(LINKCC) -D_WINDOWS -DVENV_REDIRECT -DPYTHON_EXECUTABLE_WITH_VERSION="L\"pythonw$(LDVERSION)$(EXE)\"" -mwindows -municode -static -static-libgcc -static-libstdc++ venvwlauncher.o $(srcdir)/PC/launcher.c -o $@ -lversion platform: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform @@ -806,13 +850,17 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ # This rule builds the Cygwin Python DLL and import library if configured # for a shared core library; otherwise, this rule is a noop. -$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) +$(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) python_nt.o if test -n "$(DLLLIBRARY)"; then \ $(LDSHARED) -Wl,--out-implib=$@ -o $(DLLLIBRARY) $^ \ - $(LIBS) $(MODLIBS) $(SYSLIBS); \ + $(LIBS) $(LDFLAGS_NODIST) $(MODLIBS) $(SYSLIBS) python_nt.o; \ else true; \ fi +$(ABI3DLLLIBRARY) $(ABI3LDLIBRARY): python3dll_nt.o $(srcdir)/PC/launcher.c + $(LDSHARED) -DPYTHON_DLL_NAME=\"$(DLLLIBRARY)\" $(srcdir)/PC/python3dll.c -Wl,--out-implib=$(ABI3LDLIBRARY) -o $(ABI3DLLLIBRARY) python3dll_nt.o \ + $(LDFLAGS_NODIST); + # wasm32-emscripten browser build # wasm assets directory is relative to current build dir, e.g. "./usr/local". # --preload-file turns a relative asset path into an absolute path. @@ -974,7 +1022,7 @@ BOOTSTRAP_HEADERS = \ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS) _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local - $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ + $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ -municode -mwindows $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) @@ -1276,9 +1324,15 @@ Python/dynload_hpux.o: $(srcdir)/Python/dynload_hpux.c Makefile -DSHLIB_EXT='"$(EXT_SUFFIX)"' \ -o $@ $(srcdir)/Python/dynload_hpux.c +Python/dynload_win.o: $(srcdir)/Python/dynload_win.c Makefile + $(CC) -c $(PY_CORE_CFLAGS) \ + -DPYD_PLATFORM_TAG='"$(PYD_PLATFORM_TAG)"' \ + -o $@ $(srcdir)/Python/dynload_win.c + Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile $(srcdir)/Include/pydtrace.h $(CC) -c $(PY_CORE_CFLAGS) \ -DABIFLAGS='"$(ABIFLAGS)"' \ + -DVPATH='"$(VPATH)"' \ $(MULTIARCH_CPPFLAGS) \ -o $@ $(srcdir)/Python/sysmodule.c @@ -1496,6 +1550,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/frameobject.h \ $(srcdir)/Include/import.h \ $(srcdir)/Include/intrcheck.h \ + $(srcdir)/Include/iscygpty.h \ $(srcdir)/Include/iterobject.h \ $(srcdir)/Include/listobject.h \ $(srcdir)/Include/longobject.h \ @@ -1790,7 +1845,7 @@ $(DESTSHARED): # Install the interpreter with $(VERSION) affixed # This goes into $(exec_prefix) altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ - @for i in $(BINDIR) $(LIBDIR); \ + @for i in $(BINDIR) $(LIBDIR) $(VENVLAUNCHERDIR); \ do \ if test ! -d $(DESTDIR)$$i; then \ echo "Creating directory $$i"; \ @@ -1800,6 +1855,9 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ done if test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ $(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDPYTHONW) $(DESTDIR)$(BINDIR)/python3w$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/python$(EXE); \ + $(INSTALL_PROGRAM) $(BUILDVENVWLAUNCHER) $(DESTDIR)$(VENVLAUNCHERDIR)/pythonw$(EXE); \ else \ $(INSTALL_PROGRAM) $(STRIPFLAG) Mac/pythonw $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE); \ fi @@ -1813,6 +1871,7 @@ altbininstall: $(BUILDPYTHON) @FRAMEWORKPYTHONW@ if test -f $(LDLIBRARY) && test "$(PYTHONFRAMEWORKDIR)" = "no-framework" ; then \ if test -n "$(DLLLIBRARY)" ; then \ $(INSTALL_SHARED) $(DLLLIBRARY) $(DESTDIR)$(BINDIR); \ + $(INSTALL_SHARED) $(ABI3DLLLIBRARY) $(DESTDIR)$(BINDIR); \ else \ $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(LIBDIR)/$(INSTSONAME); \ if test $(LDLIBRARY) != $(INSTSONAME); then \ @@ -1927,6 +1986,7 @@ LIBSUBDIRS= asyncio \ tkinter \ tomllib \ turtledemo \ + msilib \ unittest \ urllib \ venv venv/scripts venv/scripts/common venv/scripts/posix \ @@ -2228,8 +2288,9 @@ libainstall: all python-config @if test "$(STATIC_LIBPYTHON)" = 1; then \ if test -d $(LIBRARY); then :; else \ if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \ - if test "$(SHLIB_SUFFIX)" = .dll; then \ - $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \ + if test "$(SHLIB_SUFFIX)" = .dll -o "$(SHLIB_SUFFIX)" = .pyd; then \ + $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ + $(INSTALL_DATA) $(ABI3LDLIBRARY) $(DESTDIR)$(LIBDIR) ; \ else \ $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \ fi; \ @@ -2268,16 +2329,23 @@ libainstall: all python-config else true; \ fi +ifeq ($(shell uname -o),Msys) +DESTDIRFINAL=$(DESTDIR) +else +DESTDIRFINAL=$(DESTDIR)/ +endif + # Install the dynamically loadable modules # This goes into $(exec_prefix) sharedinstall: all + MSYS2_ARG_CONV_EXCL="--prefix=;--install-scripts=;--install-platlib=" \ $(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \ --prefix=$(prefix) \ --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ - --root=$(DESTDIR)/ - -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py - -rm -r $(DESTDIR)$(DESTSHARED)/__pycache__ + --root=$(DESTDIRFINAL) + -rm $(DESTDIRFINAL)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py + -rm -r $(DESTDIRFINAL)$(DESTSHARED)/__pycache__ # Here are a couple of targets for MacOSX again, to install a full # framework-based Python. frameworkinstall installs everything, the @@ -2469,7 +2537,7 @@ clean: clean-retain-profile fi clobber: clean - -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ + -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY)$(ABI3LDLIBRARY) $(ABI3DLLLIBRARY) \ tags TAGS \ config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform diff --git a/Misc/config_mingw b/Misc/config_mingw new file mode 100644 index 00000000000000..9be43fd3a11407 --- /dev/null +++ b/Misc/config_mingw @@ -0,0 +1,15 @@ +# configure defaults for mingw* hosts + +# mingw functions to ignore +ac_cv_func_ftruncate=ignore # implement it as _chsize + +# mingw-w64 functions to ignore +ac_cv_func_truncate=ignore +ac_cv_func_alarm=ignore + +# files to ignore +ac_cv_file__dev_ptmx=ignore #NOTE: under MSYS environment device exist +ac_cv_file__dev_ptc=no + +# force detection of winsock2 functionality - require wxp or newer +ac_cv_func_getpeername=yes diff --git a/Misc/cross_mingw32 b/Misc/cross_mingw32 new file mode 100644 index 00000000000000..03fde9e6af3ad7 --- /dev/null +++ b/Misc/cross_mingw32 @@ -0,0 +1,11 @@ +# configure defaults for mingw32 host if cross-build + +ac_cv_little_endian_double=yes +ac_cv_big_endian_double=no +ac_cv_mixed_endian_double=no + +ac_cv_tanh_preserves_zero_sign=yes + +ac_cv_wchar_t_signed=no + +ac_cv_have_size_t_format=no diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index 2602fe24c0402e..e0e048a0fac486 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -1,32 +1,44 @@ #!/bin/sh -# Keep this script in sync with python-config.in - exit_with_usage () { echo "Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - exit $1 + exit 1 } +# Really, python-config.py (and thus .sh) should be called directly, but +# sometimes software (e.g. GDB) calls python-config.sh as if it were the +# Python executable, passing python-config.py as the first argument. +# Work around that oddness by ignoring any .py passed as first arg. +case "$1" in + *.py) + shift + ;; +esac + if [ "$1" = "" ] ; then - exit_with_usage 1 + exit_with_usage fi # Returns the actual prefix where this script was installed to. installed_prefix () { - RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) - if which readlink >/dev/null 2>&1 ; then - if readlink -f "$RESULT" >/dev/null 2>&1; then - RESULT=$(readlink -f "$RESULT") - fi + local RESULT=$(dirname $(cd $(dirname "$1") && pwd -P)) + if [ $(which readlink) ] ; then + RESULT=$(readlink -f "$RESULT") + fi + # Since we don't know where the output from this script will end up + # we keep all paths in Windows-land since MSYS2 can handle that + # while native tools can't handle paths in MSYS2-land. + if [ "$OSTYPE" = "msys" ]; then + RESULT=$(cd "$RESULT" && pwd -W) fi echo $RESULT } prefix_real=$(installed_prefix "$0") -# Use sed to fix paths from their built-to locations to their installed-to +# Use sed to fix paths from their built-to locations to their installed to # locations. Keep prefix & exec_prefix using their original values in case # they are referenced in other configure variables, to prevent double # substitution, issue #22140. @@ -41,13 +53,17 @@ LIBM="@LIBM@" LIBC="@LIBC@" SYSLIBS="$LIBM $LIBC" ABIFLAGS="@ABIFLAGS@" +# Protect against lack of substitution. +if [ "$ABIFLAGS" = "@""ABIFLAGS""@" ] ; then + ABIFLAGS= +fi LIBS="@LIBPYTHON@ @LIBS@ $SYSLIBS" LIBS_EMBED="-lpython${VERSION}${ABIFLAGS} @LIBS@ $SYSLIBS" BASECFLAGS="@BASECFLAGS@" -LDLIBRARY="@LDLIBRARY@" OPT="@OPT@" PY_ENABLE_SHARED="@PY_ENABLE_SHARED@" LDVERSION="@LDVERSION@" +LDLIBRARY="@LDLIBRARY@" LIBDEST=${prefix_real}/lib/python${VERSION} LIBPL=$(echo "@LIBPL@" | sed "s#$prefix#$prefix_real#") SO="@EXT_SUFFIX@" @@ -61,7 +77,7 @@ for ARG in $* do case $ARG in --help) - exit_with_usage 0 + exit_with_usage ;; --embed) PY_EMBED=1 @@ -69,7 +85,7 @@ do --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--abiflags|--configdir) ;; *) - exit_with_usage 1 + exit_with_usage ;; esac done @@ -80,37 +96,37 @@ fi for ARG in "$@" do - case "$ARG" in + case $ARG in --prefix) - echo "$prefix_real" + echo -ne "$prefix_real" ;; --exec-prefix) - echo "$exec_prefix_real" + echo -ne "$exec_prefix_real " ;; --includes) - echo "$INCDIR $PLATINCDIR" + echo -ne "$INCDIR $PLATINCDIR" ;; --cflags) - echo "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" + echo -ne "$INCDIR $PLATINCDIR $BASECFLAGS $CFLAGS $OPT" ;; --libs) - echo "$LIBS" + echo -ne "$LIBS" ;; --ldflags) LIBPLUSED= if [ "$PY_ENABLE_SHARED" = "0" ] ; then LIBPLUSED="-L$LIBPL" fi - echo "$LIBPLUSED -L$libdir $LIBS" + echo -ne "$LIBPLUSED -L$libdir $LIBS " ;; --extension-suffix) - echo "$SO" + echo -ne "$SO " ;; --abiflags) - echo "$ABIFLAGS" + echo -ne "$ABIFLAGS " ;; --configdir) - echo "$LIBPL" + echo -ne "$LIBPL " ;; esac done diff --git a/Misc/python.pc.in b/Misc/python.pc.in index 87e04decc2a2d5..39001902558f87 100644 --- a/Misc/python.pc.in +++ b/Misc/python.pc.in @@ -9,5 +9,5 @@ Description: Build a C extension for Python Requires: Version: @VERSION@ Libs.private: @LIBS@ -Libs: +Libs: -L${libdir} -lpython@VERSION@@ABIFLAGS@ Cflags: -I${includedir}/python@VERSION@@ABIFLAGS@ diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index e3e9b96b0630df..6986290fb0f2f3 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -8,15 +8,15 @@ # module C APIs are used in core atexit atexitmodule.c faulthandler faulthandler.c -posix posixmodule.c -_signal signalmodule.c +@INITSYS@ posixmodule.c +_signal signalmodule.c -lws2_32 _tracemalloc _tracemalloc.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c _collections _collectionsmodule.c errno errnomodule.c -_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c +_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c _io/winconsoleio.c itertools itertoolsmodule.c _sre _sre/sre.c _thread _threadmodule.c @@ -33,3 +33,8 @@ _symtable symtablemodule.c # for systems without $HOME env, used by site._getuserbase() @MODULE_PWD_TRUE@pwd pwdmodule.c + +# build-in modules for windows platform: +@USE_WIN32_MODULE@winreg ../PC/winreg.c +@MODULE_MSVCRT_TRUE@msvcrt -DPy_BUILD_CORE ../PC/msvcrtmodule.c +@MODULE__WINAPI_TRUE@_winapi _winapi.c diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fc73264ff5a32c..6ed71fa5600570 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3403,6 +3403,18 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) mangled_name = alloca(strlen(name) + 1 + 1 + 1 + 3); /* \0 _ @ %d */ if (!mangled_name) return NULL; + /* Issue: for stdcall decorated export functions MSVC compiler adds + * underscore, but GCC compiler create them without. This is + * visible by example for _ctypes_test.pyd module. + * As well functions from system libraries are without underscore. + * Solutions: + * - If a python module is build with gcc option --add-stdcall-alias + * the module will contain XXX as alias for function XXX@ as result + * first search in this method will succeed. + * - Distutil may use compiler to create def-file, to modify it as + * add underscore alias and with new def file to create module. + * - Or may be just to search for function without underscore. + */ for (i = 0; i < 32; ++i) { sprintf(mangled_name, "_%s@%d", name, i*4); Py_BEGIN_ALLOW_THREADS @@ -3410,6 +3422,13 @@ static PPROC FindAddress(void *handle, const char *name, PyObject *type) Py_END_ALLOW_THREADS if (address) return address; + /* search for function without underscore as weel */ + sprintf(mangled_name, "%s@%d", name, i*4); + Py_BEGIN_ALLOW_THREADS + address = (PPROC)GetProcAddress(handle, mangled_name); + Py_END_ALLOW_THREADS + if (address) + return address; } return NULL; #endif diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index e6440fa9cd364d..f42714a410142b 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -12,7 +12,7 @@ #include #include -#if defined(WIN32) && !defined(__CYGWIN__) +#if defined(WIN32) && !defined(__CYGWIN__) && !defined(__MINGW32__) #include "gdbmerrno.h" extern const char * gdbm_strerror(gdbm_error); #endif diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 4496609afcbc26..80cc6f98f175ca 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -20,6 +20,7 @@ #endif #include /* For offsetof */ #include "_iomodule.h" +#include "iscygpty.h" /* * Known likely problems: @@ -1129,7 +1130,7 @@ _io_FileIO_isatty_impl(fileio *self) return err_closed(); Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 23c38e14d997d1..dfb6c7e607853c 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -12,6 +12,13 @@ This software comes with no warranty. Use at your own risk. #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" +#ifdef __MINGW32__ +/* The header libintl.h and library libintl may exist on mingw host. + * To be compatible with MSVC build we has to undef some defines. + */ +#undef HAVE_LIBINTL_H +#undef HAVE_BIND_TEXTDOMAIN_CODESET +#endif #include #include diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 0809c2455dbebc..bbb1b38f5a6009 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -172,7 +172,7 @@ static PyMethodDef module_methods[] = { _MULTIPROCESSING_RECV_METHODDEF _MULTIPROCESSING_SEND_METHODDEF #endif -#if !defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__) +#if defined(MS_WINDOWS) || (!defined(POSIX_SEMAPHORES_NOT_ENABLED) && !defined(__ANDROID__)) _MULTIPROCESSING_SEM_UNLINK_METHODDEF #endif {NULL} diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index 3a8314b1db8331..c07cfe82085d44 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -21,7 +21,10 @@ # endif # define SEM_HANDLE HANDLE # define SEM_VALUE_MAX LONG_MAX -# define HAVE_MP_SEMAPHORE +# define HAVE_MP_SEMAPHORE +# if defined(HAVE_SEM_OPEN) && defined(_POSIX_THREADS) +# include +# endif #else # include /* O_CREAT and O_EXCL */ # if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 7fb1f2f561b0f9..fd262ae8192a0d 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -41,7 +41,9 @@ #define WINDOWS_LEAN_AND_MEAN #include "windows.h" +#if defined(Py_DEBUG) #include +#endif #include "winreparse.h" #if defined(MS_WIN32) && !defined(MS_WIN64) @@ -969,7 +971,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) DWORD err; BOOL result; PyObject *value; - Py_ssize_t handle_list_size; + Py_ssize_t handle_list_size = 0; DWORD attribute_count = 0; SIZE_T attribute_list_size = 0; diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index e5b96be8f68da8..0e5ab450f75a27 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1765,7 +1765,7 @@ PyDoc_STRVAR(channelid_doc, "A channel ID identifies a channel and may be used as an int."); static PyTypeObject ChannelIDtype = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + PyVarObject_HEAD_INIT(NULL, 0) "_xxsubinterpreters.ChannelID", /* tp_name */ sizeof(channelid), /* tp_basicsize */ 0, /* tp_itemsize */ diff --git a/Modules/getpath.c b/Modules/getpath.c index bc730fcd7dc757..0419c2a25fbac6 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -54,6 +54,25 @@ /* HELPER FUNCTIONS for getpath.py */ +static PyObject * +getpath_normpath(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *r = NULL; + PyObject *pathobj; + wchar_t *path; + if (!PyArg_ParseTuple(args, "U", &pathobj)) { + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(pathobj, &len); + if (!buffer) { + return NULL; + } + r = PyUnicode_FromWideChar(_Py_normpath(buffer, len), -1); + PyMem_Free(buffer); + return r; +} + static PyObject * getpath_abspath(PyObject *Py_UNUSED(self), PyObject *args) { @@ -88,6 +107,12 @@ getpath_basename(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return Py_NewRef(path); } @@ -104,6 +129,12 @@ getpath_dirname(PyObject *Py_UNUSED(self), PyObject *args) } Py_ssize_t end = PyUnicode_GET_LENGTH(path); Py_ssize_t pos = PyUnicode_FindChar(path, SEP, 0, end, -1); +#ifdef ALTSEP + if (pos < 0) { + // try using altsep + pos = PyUnicode_FindChar(path, ALTSEP, 0, end, -1); + } +#endif if (pos < 0) { return PyUnicode_FromStringAndSize(NULL, 0); } @@ -513,6 +544,7 @@ getpath_realpath(PyObject *Py_UNUSED(self) , PyObject *args) static PyMethodDef getpath_methods[] = { + {"normpath", getpath_normpath, METH_VARARGS, NULL}, {"abspath", getpath_abspath, METH_VARARGS, NULL}, {"basename", getpath_basename, METH_VARARGS, NULL}, {"dirname", getpath_dirname, METH_VARARGS, NULL}, @@ -884,6 +916,11 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) #else !decode_to_dict(dict, "os_name", "posix") || #endif +#ifdef __MINGW32__ + !int_to_dict(dict, "is_mingw", 1) || +#else + !int_to_dict(dict, "is_mingw", 0) || +#endif #ifdef WITH_NEXT_FRAMEWORK !int_to_dict(dict, "WITH_NEXT_FRAMEWORK", 1) || #else @@ -910,6 +947,9 @@ _PyConfig_InitPathConfig(PyConfig *config, int compute_path_config) !funcs_to_dict(dict, config->pathconfig_warnings) || #ifndef MS_WINDOWS PyDict_SetItemString(dict, "winreg", Py_None) < 0 || +#endif +#ifdef __MINGW32__ + !env_to_dict(dict, "ENV_MSYSTEM", 0) || #endif PyDict_SetItemString(dict, "__builtins__", PyEval_GetBuiltins()) < 0 ) { diff --git a/Modules/getpath.py b/Modules/getpath.py index 74f918cf28e402..c98cb1f20d1be8 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -30,6 +30,7 @@ # ** Values known at compile time ** # os_name -- [in] one of 'nt', 'posix', 'darwin' +# is_mingw -- [in] True if targeting MinGW # PREFIX -- [in] sysconfig.get_config_var(...) # EXEC_PREFIX -- [in] sysconfig.get_config_var(...) # PYTHONPATH -- [in] sysconfig.get_config_var(...) @@ -51,6 +52,7 @@ # ENV_PYTHONHOME -- [in] getenv(...) # ENV_PYTHONEXECUTABLE -- [in] getenv(...) # ENV___PYVENV_LAUNCHER__ -- [in] getenv(...) +# ENV_MSYSTEM -- [in] getenv(...) # ** Values calculated at runtime ** # config -- [in/out] dict of the PyConfig structure @@ -185,8 +187,27 @@ ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' DELIM = ':' SEP = '/' + ALTSEP = None -elif os_name == 'nt': +elif os_name == 'nt' and is_mingw: + BUILDDIR_TXT = 'pybuilddir.txt' + BUILD_LANDMARK = 'Modules/Setup.local' + DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' + STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}' + STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] + PLATSTDLIB_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}/lib-dynload' + BUILDSTDLIB_LANDMARKS = ['Lib/os.py'] + VENV_LANDMARK = 'pyvenv.cfg' + ZIP_LANDMARK = f'{platlibdir}/python{VERSION_MAJOR}{VERSION_MINOR}.zip' + DELIM = ';' + if ENV_MSYSTEM: + SEP = '/' + ALTSEP = '\\' + else: + SEP = '\\' + ALTSEP = '/' + +elif os_name == 'nt': # MSVC BUILDDIR_TXT = 'pybuilddir.txt' BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' @@ -199,6 +220,7 @@ WINREG_KEY = f'SOFTWARE\\Python\\PythonCore\\{PYWINVER}\\PythonPath' DELIM = ';' SEP = '\\' + ALTSEP = '/' # ****************************************************************************** @@ -211,6 +233,8 @@ def search_up(prefix, *landmarks, test=isfile): return prefix prefix = dirname(prefix) +def _normpath(p): + return normpath(p) if p is not None else None # ****************************************************************************** # READ VARIABLES FROM config @@ -263,10 +287,10 @@ def search_up(prefix, *landmarks, test=isfile): if not executable: executable = real_executable -if not executable and SEP in program_name: +if not executable and (SEP in program_name or + (ALTSEP and ALTSEP in program_name)): # Resolve partial path program_name against current directory executable = abspath(program_name) - if not executable: # All platforms default to real_executable if known at this # stage. POSIX does not set this value. @@ -497,15 +521,15 @@ def search_up(prefix, *landmarks, test=isfile): except (FileNotFoundError, PermissionError): if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': + if os_name == 'nt' and not is_mingw: # QUIRK: Windows builds need platstdlib_dir to be the executable # dir. Normally the builddir marker handles this, but in this # case we need to correct manually. platstdlib_dir = real_executable_dir if build_prefix: - if os_name == 'nt': - # QUIRK: No searching for more landmarks on Windows + if os_name == 'nt' and not is_mingw: + # QUIRK: No searching for more landmarks on MSVC build_stdlib_prefix = build_prefix else: build_stdlib_prefix = search_up(build_prefix, *BUILDSTDLIB_LANDMARKS) @@ -552,6 +576,9 @@ def search_up(prefix, *landmarks, test=isfile): # First try to detect prefix by looking alongside our runtime library, if known if library and not prefix: library_dir = dirname(library) + if os_name == 'nt' and is_mingw: + # QUIRK: On Windows, mingw Python DLLs are in the bin directory + library_dir = joinpath(library_dir, '..') if ZIP_LANDMARK: if os_name == 'nt': # QUIRK: Windows does not search up for ZIP file @@ -597,7 +624,7 @@ def search_up(prefix, *landmarks, test=isfile): # Detect exec_prefix by searching from executable for the platstdlib_dir if PLATSTDLIB_LANDMARK and not exec_prefix: - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows always assumed these were the same # gh-100320: Our PYDs are assumed to be relative to the Lib directory # (that is, prefix) rather than the executable (that is, executable_dir) @@ -607,7 +634,7 @@ def search_up(prefix, *landmarks, test=isfile): if not exec_prefix and EXEC_PREFIX: exec_prefix = EXEC_PREFIX if not exec_prefix or not isdir(joinpath(exec_prefix, PLATSTDLIB_LANDMARK)): - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: If DLLs is missing on Windows, don't warn, just assume # that they're in exec_prefix if not platstdlib_dir: @@ -645,7 +672,7 @@ def search_up(prefix, *landmarks, test=isfile): if py_setpath: # If Py_SetPath was called then it overrides any existing search path - config['module_search_paths'] = py_setpath.split(DELIM) + config['module_search_paths'] = [_normpath(p) for p in py_setpath.split(DELIM)] config['module_search_paths_set'] = 1 elif not pythonpath_was_set: @@ -660,7 +687,7 @@ def search_up(prefix, *landmarks, test=isfile): pythonpath.append(abspath(p)) # Then add the default zip file - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows uses the library directory rather than the prefix if library: library_dir = dirname(library) @@ -673,7 +700,7 @@ def search_up(prefix, *landmarks, test=isfile): else: pythonpath.append(joinpath(prefix, ZIP_LANDMARK)) - if os_name == 'nt' and use_environment and winreg: + if (not is_mingw) and os_name == 'nt' and use_environment and winreg: # QUIRK: Windows also lists paths in the registry. Paths are stored # as the default value of each subkey of # {HKCU,HKLM}\Software\Python\PythonCore\{winver}\PythonPath @@ -714,7 +741,7 @@ def search_up(prefix, *landmarks, test=isfile): if not platstdlib_dir and exec_prefix: platstdlib_dir = joinpath(exec_prefix, PLATSTDLIB_LANDMARK) - if os_name == 'nt': + if os_name == 'nt' and (not is_mingw): # QUIRK: Windows generates paths differently if platstdlib_dir: pythonpath.append(platstdlib_dir) @@ -732,7 +759,7 @@ def search_up(prefix, *landmarks, test=isfile): if platstdlib_dir: pythonpath.append(platstdlib_dir) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 @@ -742,8 +769,8 @@ def search_up(prefix, *landmarks, test=isfile): # QUIRK: Non-Windows replaces prefix/exec_prefix with defaults when running # in build directory. This happens after pythonpath calculation. -if os_name != 'nt' and build_prefix: - prefix = config.get('prefix') or PREFIX +if (os_name != 'nt' or is_mingw) and build_prefix: + prefix = config.get('prefix') or abspath(PREFIX) exec_prefix = config.get('exec_prefix') or EXEC_PREFIX or prefix @@ -767,23 +794,23 @@ def search_up(prefix, *landmarks, test=isfile): warn("unsupported 'import' line in ._pth file") else: pythonpath.append(joinpath(pth_dir, line)) - config['module_search_paths'] = pythonpath + config['module_search_paths'] = [_normpath(p) for p in pythonpath] config['module_search_paths_set'] = 1 # ****************************************************************************** # UPDATE config FROM CALCULATED VALUES # ****************************************************************************** -config['program_name'] = program_name -config['home'] = home -config['executable'] = executable -config['base_executable'] = base_executable -config['prefix'] = prefix -config['exec_prefix'] = exec_prefix -config['base_prefix'] = base_prefix or prefix -config['base_exec_prefix'] = base_exec_prefix or exec_prefix +config['program_name'] = _normpath(program_name) +config['home'] = _normpath(home) +config['executable'] = _normpath(executable) +config['base_executable'] = _normpath(base_executable) +config['prefix'] = _normpath(prefix) +config['exec_prefix'] = _normpath(exec_prefix) +config['base_prefix'] = _normpath(base_prefix or prefix) +config['base_exec_prefix'] = _normpath(base_exec_prefix or exec_prefix) -config['platlibdir'] = platlibdir +config['platlibdir'] = _normpath(platlibdir) # test_embed expects empty strings, not None -config['stdlib_dir'] = stdlib_dir or '' -config['platstdlib_dir'] = platstdlib_dir or '' +config['stdlib_dir'] = _normpath(stdlib_dir or '') +config['platstdlib_dir'] = _normpath(platstdlib_dir or '') diff --git a/Modules/main.c b/Modules/main.c index 6904e3f76e8911..5c8708e52aca06 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -7,6 +7,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "iscygpty.h" /* Includes for exit_sigint() */ #include // perror() @@ -92,7 +93,7 @@ static inline int config_run_code(const PyConfig *config) static int stdin_is_interactive(const PyConfig *config) { - return (isatty(fileno(stdin)) || config->interactive); + return (isatty(fileno(stdin)) || config->interactive || is_cygpty(fileno(stdin))); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index cfadb6a06602fd..71c53f9c444298 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -50,6 +50,7 @@ #ifdef __ANDROID__ # undef HAVE_FACCESSAT #endif +#include "iscygpty.h" #include // ctermid() #include // system() @@ -350,6 +351,32 @@ corresponding Unix manual entries for more information on calls."); # define HAVE_CWAIT 1 # define HAVE_FSYNC 1 # define fsync _commit +# elif defined(__MINGW32__) /* GCC for windows hosts */ +/* getlogin is detected by configure on mingw-w64 */ +# undef HAVE_GETLOGIN +/* opendir is detected by configure on mingw-w64, and for some reason +things don't work as expected. For example, os.listdir always returns +the cwd's directory listing instead of the one specified. By +un-defining, this, os.listdir will use the one which uses native +windows API. */ +# undef HAVE_OPENDIR +/*# define HAVE_GETCWD 1 - detected by configure*/ +# define HAVE_GETPPID 1 +# define HAVE_GETLOGIN 1 +# define HAVE_SPAWNV 1 +# define HAVE_WSPAWNV 1 +# define HAVE_WEXECV 1 +/*# define HAVE_EXECV 1 - detected by configure*/ +# define HAVE_PIPE 1 +# define HAVE_POPEN 1 +# define HAVE_SYSTEM 1 +# define HAVE_CWAIT 1 +# define HAVE_FSYNC 1 +# define fsync _commit +# include +# ifndef _MAX_ENV +# define _MAX_ENV 32767 +# endif # endif /* _MSC_VER */ #endif /* ! __WATCOMC__ || __QNX__ */ @@ -428,7 +455,7 @@ extern char *ctermid_r(char *); # endif #endif -#ifdef _MSC_VER +#ifdef MS_WINDOWS # ifdef HAVE_DIRECT_H # include # endif @@ -439,7 +466,7 @@ extern char *ctermid_r(char *); # include # endif # include -#endif /* _MSC_VER */ +#endif /* MS_WINDOWS */ #ifndef MAXPATHLEN # if defined(PATH_MAX) && PATH_MAX > 1024 @@ -1593,9 +1620,9 @@ win32_get_reparse_tag(HANDLE reparse_point_handle, ULONG *reparse_tag) ** man environ(7). */ #include -#elif !defined(_MSC_VER) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) +#elif !defined(MS_WINDOWS) && (!defined(__WATCOMC__) || defined(__QNX__) || defined(__VXWORKS__)) extern char **environ; -#endif /* !_MSC_VER */ +#endif /* !MS_WINDOWS */ static PyObject * convertenviron(void) @@ -3815,6 +3842,7 @@ posix_getcwd(int use_bytes) return PyErr_SetFromWindowsErr(0); } + Py_NormalizeSepsW(wbuf2); PyObject *resobj = PyUnicode_FromWideChar(wbuf2, len); if (wbuf2 != wbuf) { PyMem_RawFree(wbuf2); @@ -4419,6 +4447,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) target_path = tmp; } + Py_NormalizeSepsW(target_path); result = PyUnicode_FromWideChar(target_path, result_length); if (result && path->narrow) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); @@ -5469,7 +5498,7 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, /*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ { #ifdef MS_WINDOWS - HANDLE hFile; + HANDLE hFile = 0; FILETIME atime, mtime; #else int result; @@ -10205,7 +10234,7 @@ os_isatty_impl(PyObject *module, int fd) int return_value; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - return_value = isatty(fd); + return_value = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS return return_value; @@ -14693,7 +14722,7 @@ os__add_dll_directory_impl(PyObject *module, path_t *path) loaded. */ Py_BEGIN_ALLOW_THREADS if (!(hKernel32 = GetModuleHandleW(L"kernel32")) || - !(AddDllDirectory = (PAddDllDirectory)GetProcAddress( + !(AddDllDirectory = (PAddDllDirectory)(void *)GetProcAddress( hKernel32, "AddDllDirectory")) || !(cookie = (*AddDllDirectory)(path->wide))) { err = GetLastError(); @@ -14743,7 +14772,7 @@ os__remove_dll_directory_impl(PyObject *module, PyObject *cookie) loaded. */ Py_BEGIN_ALLOW_THREADS if (!(hKernel32 = GetModuleHandleW(L"kernel32")) || - !(RemoveDllDirectory = (PRemoveDllDirectory)GetProcAddress( + !(RemoveDllDirectory = (PRemoveDllDirectory)(void *)GetProcAddress( hKernel32, "RemoveDllDirectory")) || !(*RemoveDllDirectory)(cookieValue)) { err = GetLastError(); diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 4eea928a2683ae..6be35826d5693f 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -146,9 +146,9 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) v = PyObject_AsFileDescriptor( o ); if (v == -1) goto finally; -#if defined(_MSC_VER) +#if defined(MS_WIN32) max = 0; /* not used for Win32 */ -#else /* !_MSC_VER */ +#else /* !MS_WIN32 */ if (!_PyIsSelectable_fd(v)) { PyErr_SetString(PyExc_ValueError, "filedescriptor out of range in select()"); @@ -156,7 +156,7 @@ seq2set(PyObject *seq, fd_set *set, pylist fd2obj[FD_SETSIZE + 1]) } if (v > max) max = v; -#endif /* _MSC_VER */ +#endif /* MS_WIN32 */ FD_SET(v, set); /* add object and its file descriptor to the list */ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index e6d983afa7da85..399c1e77b0b307 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -274,7 +274,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # endif /* Macros based on the IPPROTO enum, see: https://bugs.python.org/issue29515 */ -#ifdef MS_WINDOWS +#ifdef _MSC_VER #define IPPROTO_ICMP IPPROTO_ICMP #define IPPROTO_IGMP IPPROTO_IGMP #define IPPROTO_GGP IPPROTO_GGP @@ -305,7 +305,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #define IPPROTO_PGM IPPROTO_PGM // WinSock2 only #define IPPROTO_L2TP IPPROTO_L2TP // WinSock2 only #define IPPROTO_SCTP IPPROTO_SCTP // WinSock2 only -#endif /* MS_WINDOWS */ +#endif /* _MSC_VER */ /* Provides the IsWindows7SP1OrGreater() function */ #include @@ -404,6 +404,10 @@ remove_unusable_flags(PyObject *m) /* Do not include addrinfo.h for MSVC7 or greater. 'addrinfo' and * EAI_* constants are defined in (the already included) ws2tcpip.h. */ +#elif defined(__MINGW32__) + /* Do not include addrinfo.h as minimum supported version is + * _WIN32_WINNT >= WindowsXP(0x0501) + */ #else # include "addrinfo.h" #endif diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 1b35b11cdee6af..cff1f1d054fd30 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -68,8 +68,10 @@ struct SOCKADDR_BTH_REDEF { */ # ifdef SIO_GET_MULTICAST_FILTER # include /* for SIO_RCVALL */ +#ifndef __MINGW32__ /* resolve by configure */ # define HAVE_ADDRINFO # define HAVE_SOCKADDR_STORAGE +#endif # define HAVE_GETADDRINFO # define HAVE_GETNAMEINFO # define ENABLE_IPV6 diff --git a/Modules/timemodule.c b/Modules/timemodule.c index b8e0e481cbb5e8..eabb5fe2d0ff97 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -783,7 +783,7 @@ time_strftime(PyObject *module, PyObject *args) return NULL; } -#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) +#if defined(MS_WINDOWS) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) || defined(__VXWORKS__) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); diff --git a/Objects/fileobject.c b/Objects/fileobject.c index ffe55eb7c3730e..1d73c093123207 100644 --- a/Objects/fileobject.c +++ b/Objects/fileobject.c @@ -3,6 +3,7 @@ #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "iscygpty.h" #include "pycore_runtime.h" // _PyRuntime #if defined(HAVE_GETC_UNLOCKED) && !defined(_Py_MEMORY_SANITIZER) @@ -387,7 +388,7 @@ stdprinter_isatty(PyStdPrinter_Object *self, PyObject *Py_UNUSED(ignored)) } Py_BEGIN_ALLOW_THREADS - res = isatty(self->fd); + res = isatty(self->fd) || is_cygpty(self->fd); Py_END_ALLOW_THREADS return PyBool_FromLong(res); diff --git a/PC/_testconsole.c b/PC/_testconsole.c index a8308835d8f85d..52aca33c2eaaf4 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -10,7 +10,7 @@ #ifdef MS_WINDOWS #include "pycore_fileutils.h" // _Py_get_osfhandle() -#include "..\modules\_io\_iomodule.h" +#include "../Modules/_io/_iomodule.h" #define WIN32_LEAN_AND_MEAN #include @@ -108,7 +108,7 @@ _testconsole_read_output_impl(PyObject *module, PyObject *file) Py_RETURN_NONE; } -#include "clinic\_testconsole.c.h" +#include "clinic/_testconsole.c.h" PyMethodDef testconsole_methods[] = { _TESTCONSOLE_WRITE_INPUT_METHODDEF diff --git a/PC/launcher.c b/PC/launcher.c index da566a180168c5..09ac7d902cb950 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -155,12 +155,12 @@ static wchar_t * get_env(wchar_t * key) #else #if defined(_WINDOWS) -#define PYTHON_EXECUTABLE L"pythonw.exe" +#define PYTHON_EXECUTABLE PYTHON_EXECUTABLE_WITH_VERSION #define EXECUTABLEPATH_VALUE L"WindowedExecutablePath" #else -#define PYTHON_EXECUTABLE L"python.exe" +#define PYTHON_EXECUTABLE PYTHON_EXECUTABLE_WITH_VERSION #define EXECUTABLEPATH_VALUE L"ExecutablePath" #endif @@ -925,7 +925,7 @@ static COMMAND path_command; static COMMAND * find_on_path(wchar_t * name) { wchar_t * pathext; - size_t varsize; + size_t requiredSize; wchar_t * context = NULL; wchar_t * extension; COMMAND * result = NULL; @@ -942,18 +942,23 @@ static COMMAND * find_on_path(wchar_t * name) } else { /* No extension - search using registered extensions. */ - rc = _wdupenv_s(&pathext, &varsize, L"PATHEXT"); - if (rc == 0) { - extension = wcstok_s(pathext, L";", &context); - while (extension) { - len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); - if (len) { - result = &path_command; - break; + _wgetenv_s(&requiredSize, NULL, 0, L"PATHEXT"); + if (requiredSize > 0) { + pathext = (wchar_t *)malloc(requiredSize * sizeof(wchar_t)); + /* No extension - search using registered extensions. */ + rc = _wgetenv_s(&requiredSize, pathext, requiredSize, L"PATHEXT"); + if (rc == 0) { + extension = wcstok_s(pathext, L";", &context); + while (extension) { + len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); + if (len) { + result = &path_command; + break; + } + extension = wcstok_s(NULL, L";", &context); } - extension = wcstok_s(NULL, L";", &context); + free(pathext); } - free(pathext); } } return result; @@ -1913,7 +1918,7 @@ process(int argc, wchar_t ** argv) if (_wfopen_s(&f, venv_cfg_path, L"r")) { error(RC_BAD_VENV_CFG, L"Cannot read '%ls'", venv_cfg_path); } - cb = fread_s(buffer, sizeof(buffer), sizeof(buffer[0]), + cb = fread(buffer, sizeof(buffer[0]), sizeof(buffer) / sizeof(buffer[0]), f); fclose(f); diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 1f78d99c790ff9..0f2da8055f0553 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -22,7 +22,9 @@ #include #include #include +#ifdef _DEBUG #include +#endif #include #ifdef _MSC_VER diff --git a/PC/pylauncher.rc b/PC/pylauncher.rc index 11862643aa6989..84b291722f6633 100644 --- a/PC/pylauncher.rc +++ b/PC/pylauncher.rc @@ -12,17 +12,17 @@ 1 RT_MANIFEST "python.manifest" #if defined(PY_ICON) -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" #elif defined(PYW_ICON) -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" #else -1 ICON DISCARDABLE "icons\launcher.ico" -2 ICON DISCARDABLE "icons\py.ico" -3 ICON DISCARDABLE "icons\pyc.ico" -4 ICON DISCARDABLE "icons\pyd.ico" -5 ICON DISCARDABLE "icons\python.ico" -6 ICON DISCARDABLE "icons\pythonw.ico" -7 ICON DISCARDABLE "icons\setup.ico" +1 ICON DISCARDABLE "icons/launcher.ico" +2 ICON DISCARDABLE "icons/py.ico" +3 ICON DISCARDABLE "icons/pyc.ico" +4 ICON DISCARDABLE "icons/pyd.ico" +5 ICON DISCARDABLE "icons/python.ico" +6 ICON DISCARDABLE "icons/pythonw.ico" +7 ICON DISCARDABLE "icons/setup.ico" #endif 1 USAGE "launcher-usage.txt" @@ -64,4 +64,4 @@ BEGIN BEGIN VALUE "Translation", 0x0, 1200 END -END \ No newline at end of file +END diff --git a/PC/python3dll.c b/PC/python3dll.c index 50e7a9607bec95..8b524fd40e0368 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -3,6 +3,7 @@ /* Generated by Tools/scripts/stable_abi.py */ +#ifdef _MSC_VER #ifdef _M_IX86 #define DECORATE "_" #else @@ -13,6 +14,13 @@ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name)) #define EXPORT_DATA(name) \ __pragma(comment(linker, "/EXPORT:" DECORATE #name "=" PYTHON_DLL_NAME "." #name ",DATA")) +#else +// XXX: Why do we need the .dll extension and no DECORATE compared to the MSVC case? +#define EXPORT_FUNC(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\" \""); +#define EXPORT_DATA(name) \ + asm(".section .drectve\n\t.ascii \" -export:" #name "=\\\"" PYTHON_DLL_NAME "." #name "\\\",DATA \""); +#endif EXPORT_FUNC(_Py_BuildValue_SizeT) EXPORT_FUNC(_Py_CheckRecursiveCall) diff --git a/PC/python_exe.rc b/PC/python_exe.rc index c3d3bff019895e..dde0e5384201fa 100644 --- a/PC/python_exe.rc +++ b/PC/python_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\python.ico" +1 ICON DISCARDABLE "icons/python.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/pythonw_exe.rc b/PC/pythonw_exe.rc index 38570b74fa3e02..7ce1043298eac4 100644 --- a/PC/pythonw_exe.rc +++ b/PC/pythonw_exe.rc @@ -12,7 +12,7 @@ // current versions of Windows. 1 RT_MANIFEST "python.manifest" -1 ICON DISCARDABLE "icons\pythonw.ico" +1 ICON DISCARDABLE "icons/pythonw.ico" ///////////////////////////////////////////////////////////////////////////// diff --git a/PC/winreg.c b/PC/winreg.c index 940278194f4cf5..80c1ef17433a2a 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -18,6 +18,25 @@ #include "structmember.h" // PyMemberDef #include +#ifndef SIZEOF_HKEY +/* used only here */ +#if defined(MS_WIN64) +# define SIZEOF_HKEY 8 +#elif defined(MS_WIN32) +# define SIZEOF_HKEY 4 +#else +# error "SIZEOF_HKEY is not defined" +#endif +#endif + +#ifndef REG_LEGAL_CHANGE_FILTER +#define REG_LEGAL_CHANGE_FILTER (\ + REG_NOTIFY_CHANGE_NAME |\ + REG_NOTIFY_CHANGE_ATTRIBUTES |\ + REG_NOTIFY_CHANGE_LAST_SET |\ + REG_NOTIFY_CHANGE_SECURITY ) +#endif + static BOOL PyHKEY_AsHKEY(PyObject *ob, HKEY *pRes, BOOL bNoneOK); static BOOL clinic_HKEY_converter(PyObject *ob, void *p); static PyObject *PyHKEY_FromHKEY(HKEY h); @@ -806,6 +825,7 @@ Reg2Py(BYTE *retDataBuf, DWORD retDataSize, DWORD typ) case REG_BINARY: /* ALSO handle ALL unknown data types here. Even if we can't support it natively, we should handle the bits. */ + /* fallthrough */ default: if (retDataSize == 0) { Py_INCREF(Py_None); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index f30040507bcb33..090f598e64d2ae 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1,6 +1,7 @@ /* Built-in functions */ #include "Python.h" +#include "iscygpty.h" #include #include "pycore_ast.h" // _PyAST_Validate() #include "pycore_call.h" // _PyObject_CallNoArgs() @@ -2135,7 +2136,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_DECREF(tmp); if (fd < 0 && PyErr_Occurred()) return NULL; - tty = fd == fileno(stdin) && isatty(fd); + tty = fd == fileno(stdin) && (isatty(fd) || is_cygpty(fd)); } if (tty) { tmp = PyObject_CallMethodNoArgs(fout, &_Py_ID(fileno)); @@ -2148,7 +2149,7 @@ builtin_input_impl(PyObject *module, PyObject *prompt) Py_DECREF(tmp); if (fd < 0 && PyErr_Occurred()) return NULL; - tty = fd == fileno(stdout) && isatty(fd); + tty = fd == fileno(stdout) && (isatty(fd) || is_cygpty(fd)); } } diff --git a/Python/dynamic_annotations.c b/Python/dynamic_annotations.c index 7febaa09df1950..70d5b3dc72069c 100644 --- a/Python/dynamic_annotations.c +++ b/Python/dynamic_annotations.c @@ -27,7 +27,7 @@ * Author: Kostya Serebryany */ -#ifdef _MSC_VER +#ifdef MS_WINDOWS # include #endif diff --git a/Python/dynload_win.c b/Python/dynload_win.c index 5dc4095237a6be..fcb732148a48f6 100644 --- a/Python/dynload_win.c +++ b/Python/dynload_win.c @@ -4,6 +4,7 @@ #include "Python.h" #include "pycore_fileutils.h" // _Py_add_relfile() #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_initconfig.h" #ifdef HAVE_DIRECT_H #include @@ -170,8 +171,7 @@ static char *GetPythonImport (HINSTANCE hModule) Return whether the DLL was found. */ extern HMODULE PyWin_DLLhModule; -static int -_Py_CheckPython3(void) +int _Py_CheckPython3(void) { static int python3_checked = 0; static HANDLE hPython3; @@ -224,7 +224,21 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, dl_funcptr p; char funcname[258], *import_python; - _Py_CheckPython3(); + int use_legacy = 0; + DWORD load_library_flags = 0; + + _Py_get_env_flag(1, &use_legacy, "PYTHONLEGACYWINDOWSDLLLOADING"); + + if (use_legacy) { + load_library_flags = LOAD_WITH_ALTERED_SEARCH_PATH; + } else { + load_library_flags = LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR; + } + +#ifdef _MSC_VER + _Py_CheckPython3(); +#endif #if USE_UNICODE_WCHAR_CACHE const wchar_t *wpathname = _PyUnicode_AsUnicode(pathname); @@ -248,9 +262,7 @@ dl_funcptr _PyImport_FindSharedFuncptrWindows(const char *prefix, AddDllDirectory function. We add SEARCH_DLL_LOAD_DIR to ensure DLLs adjacent to the PYD are preferred. */ Py_BEGIN_ALLOW_THREADS - hDLL = LoadLibraryExW(wpathname, NULL, - LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | - LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR); + hDLL = LoadLibraryExW(wpathname, NULL, load_library_flags); Py_END_ALLOW_THREADS #if !USE_UNICODE_WCHAR_CACHE PyMem_Free(wpathname); diff --git a/Python/fileutils.c b/Python/fileutils.c index b310a6c077d91e..c575e5a91c1551 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "iscygpty.h" #include "pycore_fileutils.h" // fileutils definitions #include "pycore_runtime.h" // _PyRuntime #include "osdefs.h" // SEP @@ -71,7 +72,7 @@ _Py_device_encoding(int fd) int valid; Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH - valid = isatty(fd); + valid = isatty(fd) || is_cygpty(fd); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS if (!valid) @@ -1809,12 +1810,12 @@ _Py_write_impl(int fd, const void *buf, size_t count, int gil_held) depending on heap usage). */ if (gil_held) { Py_BEGIN_ALLOW_THREADS - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } Py_END_ALLOW_THREADS } else { - if (isatty(fd)) { + if (isatty(fd) || is_cygpty(fd)) { count = 32767; } } @@ -2004,19 +2005,31 @@ int _Py_isabs(const wchar_t *path) { #ifdef MS_WINDOWS + // create a copy of path and replace all forward slashes with backslashes + // pathccskiproot does not handle forward slashes + wchar_t *path_copy = _wcsdup(path); + if (path_copy == NULL) { + return 0; + } + Py_NormalizeSepsPathcchW(path_copy); + const wchar_t *tail; - HRESULT hr = PathCchSkipRoot(path, &tail); - if (FAILED(hr) || path == tail) { + HRESULT hr = PathCchSkipRoot(path_copy, &tail); + if (FAILED(hr) || path_copy == tail) { + free(path_copy); return 0; } - if (tail == &path[1] && (path[0] == SEP || path[0] == ALTSEP)) { + if (tail == &path_copy[1] && (path_copy[0] == SEP || path_copy[0] == ALTSEP)) { // Exclude paths with leading SEP + free(path_copy); return 0; } - if (tail == &path[2] && path[1] == L':') { + if (tail == &path_copy[2] && path_copy[1] == L':') { // Exclude drive-relative paths (e.g. C:filename.ext) + free(path_copy); return 0; } + free(path_copy); return 1; #else return (path[0] == SEP); @@ -2049,7 +2062,11 @@ _Py_abspath(const wchar_t *path, wchar_t **abspath_p) } #ifdef MS_WINDOWS - return _PyOS_getfullpathname(path, abspath_p); + if (_PyOS_getfullpathname(path, abspath_p) < 0){ + return -1; + } + *abspath_p = _Py_normpath(*abspath_p, -1); + return 0; #else wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; @@ -2093,6 +2110,8 @@ join_relfile(wchar_t *buffer, size_t bufsize, const wchar_t *dirname, const wchar_t *relfile) { #ifdef MS_WINDOWS + Py_NormalizeSepsPathcchW(dirname); + Py_NormalizeSepsPathcchW(relfile); if (FAILED(PathCchCombineEx(buffer, bufsize, dirname, relfile, PATHCCH_ALLOW_LONG_PATHS))) { return -1; @@ -2195,11 +2214,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) wchar_t *minP2 = path; // the beginning of the destination range wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases + const wchar_t sep = Py_GetSepW(NULL); +#ifdef ALTSEP + const wchar_t altsep = Py_GetAltSepW(NULL); +#endif + #define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) #ifdef ALTSEP -#define IS_SEP(x) (*(x) == SEP || *(x) == ALTSEP) +#define IS_SEP(x) (*(x) == sep || *(x) == altsep) #else -#define IS_SEP(x) (*(x) == SEP) +#define IS_SEP(x) (*(x) == sep) #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) @@ -2210,7 +2234,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) path++; } p1 = p2 = minP2 = path; - lastC = SEP; + lastC = sep; } #ifdef MS_WINDOWS // Skip past drive segment and update minP2 @@ -2224,13 +2248,13 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) // and network paths, including the first segment. else if (IS_SEP(&p1[0]) && IS_SEP(&p1[1])) { int sepCount = 2; - *p2++ = SEP; - *p2++ = SEP; + *p2++ = sep; + *p2++ = sep; p1 += 2; for (; !IS_END(p1) && sepCount; ++p1) { if (IS_SEP(p1)) { --sepCount; - *p2++ = lastC = SEP; + *p2++ = lastC = sep; } else { *p2++ = lastC = *p1; } @@ -2243,7 +2267,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = *p1++; *p2++ = *p1++; minP2 = p2 - 1; // Absolute path has SEP at minP2 - lastC = SEP; + lastC = sep; } #endif /* MS_WINDOWS */ @@ -2251,18 +2275,18 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) for (; !IS_END(p1); ++p1) { wchar_t c = *p1; #ifdef ALTSEP - if (c == ALTSEP) { - c = SEP; + if (c == altsep) { + c = sep; } #endif - if (lastC == SEP) { + if (lastC == sep) { if (c == L'.') { int sep_at_1 = SEP_OR_END(&p1[1]); int sep_at_2 = !sep_at_1 && SEP_OR_END(&p1[2]); if (sep_at_2 && p1[1] == L'.') { wchar_t *p3 = p2; - while (p3 != minP2 && *--p3 == SEP) { } - while (p3 != minP2 && *(p3 - 1) != SEP) { --p3; } + while (p3 != minP2 && *--p3 == sep) { } + while (p3 != minP2 && *(p3 - 1) != sep) { --p3; } if (p2 == minP2 || (p3[0] == L'.' && p3[1] == L'.' && IS_SEP(&p3[2]))) { @@ -2271,7 +2295,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) *p2++ = L'.'; *p2++ = L'.'; lastC = L'.'; - } else if (p3[0] == SEP) { + } else if (p3[0] == sep) { // Absolute path, so absorb segment p2 = p3 + 1; } else { @@ -2282,7 +2306,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } else { *p2++ = lastC = c; } - } else if (c == SEP) { + } else if (c == sep) { } else { *p2++ = lastC = c; } @@ -2292,7 +2316,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } *p2 = L'\0'; if (p2 != minP2) { - while (--p2 != minP2 && *p2 == SEP) { + while (--p2 != minP2 && *p2 == sep) { *p2 = L'\0'; } } else { diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 8743e082b4ff8f..c6f5f195792ede 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_runtime.h" // _PyRuntime_Initialize() #include +#include "iscygpty.h" #ifdef MS_WINDOWS extern void PyWinFreeze_ExeInit(void); @@ -71,7 +72,7 @@ Py_FrozenMain(int argc, char **argv) sts = 0; } - if (inspect && isatty((int)fileno(stdin))) { + if (inspect && (isatty((int)fileno(stdin)) || is_cygpty((int)fileno(stdin)))) sts = PyRun_AnyFile(stdin, "") != 0; } diff --git a/Python/getcompiler.c b/Python/getcompiler.c index a5d26239e8772e..4b0b9b328ba2f8 100644 --- a/Python/getcompiler.c +++ b/Python/getcompiler.c @@ -7,10 +7,40 @@ // Note the __clang__ conditional has to come before the __GNUC__ one because // clang pretends to be GCC. -#if defined(__clang__) +#if defined(__clang__) && !defined(_WIN32) #define COMPILER "[Clang " __clang_version__ "]" #elif defined(__GNUC__) -#define COMPILER "[GCC " __VERSION__ "]" +/* To not break compatibility with things that determine + CPU arch by calling get_build_version in msvccompiler.py + (such as NumPy) add "32 bit" or "64 bit (AMD64)" on Windows + and also use a space as a separator rather than a newline. */ +#if defined(_WIN32) +#define COMP_SEP " " +#if defined(__x86_64__) +#define ARCH_SUFFIX " 64 bit (AMD64)" +#elif defined(__aarch64__) +#define ARCH_SUFFIX " 64 bit (ARM64)" +#elif defined(__arm__) +#define ARCH_SUFFIX " 32 bit (ARM)" +#else +#define ARCH_SUFFIX " 32 bit" +#endif +#else +#define COMP_SEP "\n" +#define ARCH_SUFFIX "" +#endif +#if defined(__clang__) +#define str(x) #x +#define xstr(x) str(x) +#define COMPILER COMP_SEP "[GCC Clang " xstr(__clang_major__) "." \ + xstr(__clang_minor__) "." xstr(__clang_patchlevel__) ARCH_SUFFIX "]" +#else +#if defined(_UCRT) +#define COMPILER COMP_SEP "[GCC UCRT " __VERSION__ ARCH_SUFFIX "]" +#else +#define COMPILER COMP_SEP "[GCC " __VERSION__ ARCH_SUFFIX "]" +#endif +#endif // Generic fallbacks. #elif defined(__cplusplus) #define COMPILER "[C++]" diff --git a/Python/initconfig.c b/Python/initconfig.c index 50d2498a330c36..e37682fa39c2c1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -176,7 +176,7 @@ static const char usage_envvars[] = "PYTHONVERBOSE : trace import statements (-v)\n" "PYTHONWARNINGS=arg : warning control (-W arg)\n"; -#if defined(MS_WINDOWS) +#if defined(_MSC_VER) # define PYTHONHOMEHELP "\\python{major}{minor}" #else # define PYTHONHOMEHELP "/lib/pythonX.X" diff --git a/Python/iscygpty.c b/Python/iscygpty.c new file mode 100644 index 00000000000000..722f88f2f46dcb --- /dev/null +++ b/Python/iscygpty.c @@ -0,0 +1,185 @@ +/* + * iscygpty.c -- part of ptycheck + * https://github.com/k-takata/ptycheck + * + * Copyright (c) 2015-2017 K.Takata + * + * You can redistribute it and/or modify it under the terms of either + * the MIT license (as described below) or the Vim license. + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef _WIN32 + +#include +#include +#include +#include + +#ifdef USE_FILEEXTD +/* VC 7.1 or earlier doesn't support SAL. */ +# if !defined(_MSC_VER) || (_MSC_VER < 1400) +# define __out +# define __in +# define __in_opt +# endif +/* Win32 FileID API Library: + * http://www.microsoft.com/en-us/download/details.aspx?id=22599 + * Needed for WinXP. */ +# include +#else /* USE_FILEEXTD */ +/* VC 8 or earlier. */ +# if defined(_MSC_VER) && (_MSC_VER < 1500) +# ifdef ENABLE_STUB_IMPL +# define STUB_IMPL +# else +# error "Win32 FileID API Library is required for VC2005 or earlier." +# endif +# endif +#endif /* USE_FILEEXTD */ + + +#include "iscygpty.h" + +//#define USE_DYNFILEID +#ifdef USE_DYNFILEID +typedef BOOL (WINAPI *pfnGetFileInformationByHandleEx)( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize +); +static pfnGetFileInformationByHandleEx pGetFileInformationByHandleEx = NULL; + +# ifndef USE_FILEEXTD +static BOOL WINAPI stub_GetFileInformationByHandleEx( + HANDLE hFile, + FILE_INFO_BY_HANDLE_CLASS FileInformationClass, + LPVOID lpFileInformation, + DWORD dwBufferSize + ) +{ + return FALSE; +} +# endif + +static void setup_fileid_api(void) +{ + if (pGetFileInformationByHandleEx != NULL) { + return; + } + pGetFileInformationByHandleEx = (pfnGetFileInformationByHandleEx) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), + "GetFileInformationByHandleEx"); + if (pGetFileInformationByHandleEx == NULL) { +# ifdef USE_FILEEXTD + pGetFileInformationByHandleEx = GetFileInformationByHandleEx; +# else + pGetFileInformationByHandleEx = stub_GetFileInformationByHandleEx; +# endif + } +} +#else +# define pGetFileInformationByHandleEx GetFileInformationByHandleEx +# define setup_fileid_api() +#endif + + +#define is_wprefix(s, prefix) \ + (wcsncmp((s), (prefix), sizeof(prefix) / sizeof(WCHAR) - 1) == 0) + +/* Check if the fd is a cygwin/msys's pty. */ +int is_cygpty(int fd) +{ +#ifdef STUB_IMPL + return 0; +#else + HANDLE h; + int size = sizeof(FILE_NAME_INFO) + sizeof(WCHAR) * (MAX_PATH - 1); + FILE_NAME_INFO *nameinfo; + WCHAR *p = NULL; + + setup_fileid_api(); + + h = (HANDLE) _get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) { + return 0; + } + /* Cygwin/msys's pty is a pipe. */ + if (GetFileType(h) != FILE_TYPE_PIPE) { + return 0; + } + nameinfo = malloc(size + sizeof(WCHAR)); + if (nameinfo == NULL) { + return 0; + } + /* Check the name of the pipe: + * '\{cygwin,msys}-XXXXXXXXXXXXXXXX-ptyN-{from,to}-master' */ + if (pGetFileInformationByHandleEx(h, FileNameInfo, nameinfo, size)) { + nameinfo->FileName[nameinfo->FileNameLength / sizeof(WCHAR)] = L'\0'; + p = nameinfo->FileName; + if (is_wprefix(p, L"\\cygwin-")) { /* Cygwin */ + p += 8; + } else if (is_wprefix(p, L"\\msys-")) { /* MSYS and MSYS2 */ + p += 6; + } else { + p = NULL; + } + if (p != NULL) { + while (*p && isxdigit(*p)) /* Skip 16-digit hexadecimal. */ + ++p; + if (is_wprefix(p, L"-pty")) { + p += 4; + } else { + p = NULL; + } + } + if (p != NULL) { + while (*p && isdigit(*p)) /* Skip pty number. */ + ++p; + if (is_wprefix(p, L"-from-master")) { + //p += 12; + } else if (is_wprefix(p, L"-to-master")) { + //p += 10; + } else { + p = NULL; + } + } + } + free(nameinfo); + return (p != NULL); +#endif /* STUB_IMPL */ +} + +/* Check if at least one cygwin/msys pty is used. */ +int is_cygpty_used(void) +{ + int fd, ret = 0; + + for (fd = 0; fd < 3; fd++) { + ret |= is_cygpty(fd); + } + return ret; +} + +#endif /* _WIN32 */ + +/* vim: set ts=4 sw=4: */ diff --git a/Python/pathconfig.c b/Python/pathconfig.c index be0f97c4b204a9..7eb9006be5b40a 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -2,7 +2,7 @@ #include "Python.h" #include "marshal.h" // PyMarshal_ReadObjectFromString -#include "osdefs.h" // DELIM +#include "osdefs.h" // DELIM, SEP #include "pycore_initconfig.h" #include "pycore_fileutils.h" #include "pycore_pathconfig.h" @@ -18,6 +18,158 @@ extern "C" { #endif +#ifdef __MINGW32__ +#define wcstok wcstok_s +#include +#endif + +static int +Py_StartsWithA(const char * str, const char * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +static int +Py_StartsWithW(const wchar_t * str, const wchar_t * prefix) +{ + while(*prefix) + { + if(*prefix++ != *str++) + return 0; + } + + return 1; +} + +char +Py_GetSepA(const char *name) +{ + static char sep = '\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithA(name, "\\\\?\\") != 0) + { + return '\\'; + } +#endif + if (sep != '\0') + return sep; +#if defined(__MINGW32__) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) + sep = '/'; + else + sep = '\\'; +#else + sep = SEP; +#endif + return sep; +} + +static char +Py_GetAltSepA(const char *name) +{ + char sep = Py_GetSepA(name); + if (sep == '/') + return '\\'; + return '/'; +} + +void +Py_NormalizeSepsA(char *name) +{ + assert(name != NULL); + char sep = Py_GetSepA(name); + char altsep = Py_GetAltSepA(name); + char* seps; + if (name[0] != '\0' && name[1] == ':') { + name[0] = toupper(name[0]); + } + seps = strchr(name, altsep); + while(seps) { + *seps = sep; + seps = strchr(seps, altsep); + } +} + +wchar_t +Py_GetSepW(const wchar_t *name) +{ + static wchar_t sep = L'\0'; +#ifdef _WIN32 + /* https://msdn.microsoft.com/en-gb/library/windows/desktop/aa365247%28v=vs.85%29.aspx + * The "\\?\" prefix .. indicate that the path should be passed to the system with minimal + * modification, which means that you cannot use forward slashes to represent path separators + */ + if (name != NULL && Py_StartsWithW(name, L"\\\\?\\") != 0) + { + return L'\\'; + } +#endif + if (sep != L'\0') + return sep; +#if defined(__MINGW32__) + char* msystem = getenv("MSYSTEM"); + if (msystem != NULL && strcmp(msystem, "") != 0) + sep = L'/'; + else + sep = L'\\'; +#else + sep = SEP; +#endif + return sep; +} + +wchar_t +Py_GetAltSepW(const wchar_t *name) +{ + char sep = Py_GetSepW(name); + if (sep == L'/') + return L'\\'; + return L'/'; +} + +void +Py_NormalizeSepsW(wchar_t *name) +{ + assert(name != NULL); + wchar_t sep = Py_GetSepW(name); + wchar_t altsep = Py_GetAltSepW(name); + wchar_t* seps; + if (name[0] != L'\0' && name[1] == L':') { + name[0] = towupper(name[0]); + } + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +} + +void +Py_NormalizeSepsPathcchW(wchar_t *name) +{ +#ifdef MS_WINDOWS + assert(name != NULL); + wchar_t sep = '\\'; + wchar_t altsep = '/'; + wchar_t* seps; + seps = wcschr(name, altsep); + while(seps) { + *seps = sep; + seps = wcschr(seps, altsep); + } +#endif +} /* External interface */ @@ -317,6 +469,7 @@ _Py_SetProgramFullPath(const wchar_t *program_full_path) if (has_value && _Py_path_config.program_full_path == NULL) { path_out_of_memory(__func__); } + Py_NormalizeSepsW(_Py_path_config.program_name); } @@ -509,7 +662,7 @@ _PyPathConfig_ComputeSysPath0(const PyWideStringList *argv, PyObject **path0_p) } #endif /* All others */ - PyObject *path0_obj = PyUnicode_FromWideChar(path0, n); + PyObject *path0_obj = PyUnicode_FromWideChar(_Py_normpath(path0, -1), n); if (path0_obj == NULL) { return -1; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9248e971d9c78f..1d53e12a9765e7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -31,6 +31,7 @@ extern void _PyIO_Fini(void); +#include "iscygpty.h" #include // setlocale() #include // getenv() @@ -2954,7 +2955,7 @@ Py_Exit(int sts) int Py_FdIsInteractive(FILE *fp, const char *filename) { - if (isatty((int)fileno(fp))) + if (isatty((int)fileno(fp)) || is_cygpty((int)fileno(fp))) return 1; if (!Py_InteractiveFlag) return 0; @@ -2967,7 +2968,7 @@ Py_FdIsInteractive(FILE *fp, const char *filename) int _Py_FdIsInteractive(FILE *fp, PyObject *filename) { - if (isatty((int)fileno(fp))) { + if (isatty((int)fileno(fp)) || is_cygpty((int)fileno(fp))) { return 1; } if (!Py_InteractiveFlag) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 6e9ec90c9d1c3e..1b377ca0ccbe6f 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -43,7 +43,7 @@ Data members: #include #endif /* MS_WINDOWS */ -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) extern void *PyWin_DLLhModule; /* A string loaded from the DLL at startup: */ extern const char *PyWin_DLLVersionString; @@ -2929,7 +2929,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS_FROM_STRING("byteorder", "little"); #endif -#ifdef MS_COREDLL +#if defined(MS_WINDOWS) && defined(Py_ENABLE_SHARED) SET_SYS("dllhandle", PyLong_FromVoidPtr(PyWin_DLLhModule)); SET_SYS_FROM_STRING("winver", PyWin_DLLVersionString); #endif diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 084bd587314876..f8a6765280c722 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -360,8 +360,9 @@ PyThread_release_lock(PyThread_type_lock aLock) { dprintf(("%lu: PyThread_release_lock(%p) called\n", PyThread_get_thread_ident(),aLock)); - if (!(aLock && LeaveNonRecursiveMutex((PNRMUTEX) aLock))) + if (!(aLock && LeaveNonRecursiveMutex((PNRMUTEX) aLock))) { dprintf(("%lu: Could not PyThread_release_lock(%p) error: %ld\n", PyThread_get_thread_ident(), aLock, GetLastError())); + } } /* minimum/maximum thread stack sizes supported */ diff --git a/Python/traceback.c b/Python/traceback.c index 7f47349a27a1ae..23fda627582a79 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -323,7 +323,7 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * filepath = PyBytes_AS_STRING(filebytes); /* Search tail of filename in sys.path before giving up */ - tail = strrchr(filepath, SEP); + tail = strrchr(filepath, Py_GetSepA(filepath)); if (tail == NULL) tail = filepath; else diff --git a/configure.ac b/configure.ac index e1cbb7c7fbe9d9..bb5995b744c399 100644 --- a/configure.ac +++ b/configure.ac @@ -202,9 +202,11 @@ AC_SUBST([FREEZE_MODULE]) AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) +NATIVE_PYTHON_SEARCH_PATH_MINGW=`echo $host | grep -Eq 'mingw*' && echo "$MINGW_PREFIX/bin" || echo $PATH` AC_CHECK_PROGS([PYTHON_FOR_REGEN], [python$PACKAGE_VERSION python3.10 python3.9 python3.8 python3.7 python3.6 python3 python], - [python3]) + [python3], + [$NATIVE_PYTHON_SEARCH_PATH_MINGW]) AC_SUBST(PYTHON_FOR_REGEN) AC_MSG_CHECKING([Python for regen version]) @@ -545,6 +547,9 @@ then *-*-cygwin*) ac_sys_system=Cygwin ;; + *-*-mingw*) + ac_sys_system=MINGW + ;; *-*-vxworks*) ac_sys_system=VxWorks ;; @@ -580,6 +585,7 @@ then linux*) MACHDEP="linux";; cygwin*) MACHDEP="cygwin";; darwin*) MACHDEP="darwin";; + mingw*) MACHDEP="win32";; '') MACHDEP="unknown";; esac fi @@ -605,6 +611,9 @@ if test "$cross_compiling" = yes; then ;; wasm32-*-* | wasm64-*-*) _host_cpu=$host_cpu + ;; + *-*-mingw*) + _host_cpu= ;; *) # for now, limit cross builds to known configurations @@ -612,6 +621,14 @@ if test "$cross_compiling" = yes; then AC_MSG_ERROR([cross build not supported for $host]) esac _PYTHON_HOST_PLATFORM="$MACHDEP${_host_cpu:+-$_host_cpu}" + + case "$host_os" in + mingw*) + # As sys.platform() return 'win32' to build python and extantions + # we will use 'mingw' (in setup.py and etc.) + _PYTHON_HOST_PLATFORM=mingw + ;; + esac fi # Some systems cannot stand _XOPEN_SOURCE being defined at all; they @@ -723,6 +740,65 @@ then AC_DEFINE(_INCLUDE__STDC_A1_SOURCE, 1, Define to include mbstate_t for mbrtowc) fi +# On 'semi-native' build systems (MSYS*/Cygwin targeting MinGW-w64) +# _sysconfigdata.py will contain paths that are correct only in the +# build environment. This means external modules will fail to build +# without setting up the same env and also that the build of Python +# itself will fail as the paths are not correct for the host tools. +# +# To work around these issues a set of _b2h variables are created: +# prefix_b2h, srcdir_b2h, abs_srcdir_b2h +# and abs_builddir_b2h +# .. where b2h stands for build to host. sysconfig.py replaces path +# prefixes matching the non-b2h versions with the b2h equivalents. +# +# (note this assumes the host compilers are native and *not* cross +# - in the 'semi-native' scenario only that is.) + +AC_DEFUN([ABS_PATH_HOST], +[$1=$(cd $$2 && pwd) + case $build_os in + mingw*) + case $host_os in + mingw*) $1=$(cd $$2 && pwd -W) ;; + *) ;; + esac + ;; + cygwin*) + case $host_os in + mingw*) $1=$(cygpath -w -m $$2) ;; + *) ;; + esac + ;; + esac +AC_SUBST([$1]) +]) + +AC_MSG_CHECKING(absolute host location of prefix) +ABS_PATH_HOST([prefix_b2h],[prefix]) +AC_MSG_RESULT([$prefix_b2h]) + +AC_MSG_CHECKING(absolute host location of srcdir) +ABS_PATH_HOST([srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$srcdir_b2h]) + +AC_MSG_CHECKING(absolute host location of abs_srcdir) +ABS_PATH_HOST([abs_srcdir_b2h],[srcdir]) +AC_MSG_RESULT([$abs_srcdir_b2h]) + +my_builddir=. +AC_MSG_CHECKING(Absolute host location of abs_builddir) +ABS_PATH_HOST([abs_builddir_b2h],[my_builddir]) +AC_MSG_RESULT([$abs_builddir_b2h]) + +AC_MSG_CHECKING([for init system calls]) +AC_SUBST(INITSYS) +case $host in + *-*-mingw*) INITSYS=nt;; + *) INITSYS=posix;; +esac +AC_MSG_RESULT([$INITSYS]) + # Record the configure-time value of MACOSX_DEPLOYMENT_TARGET, # it may influence the way we can build extensions, so distutils # needs to check it @@ -1163,6 +1239,28 @@ AC_CACHE_CHECK([for -Wl,--no-as-needed], [ac_cv_wl_no_as_needed], [ ]) AC_SUBST(NO_AS_NEEDED) +# initialize default configuration +py_config= +case $host in + *-*-mingw*) py_config=mingw ;; +esac +if test -n "$py_config" ; then + AC_MSG_NOTICE([loading configure defaults from .../Misc/config_$py_config"]) + . "$srcdir/Misc/config_$py_config" +fi + +# initialize defaults for cross-builds +if test "$cross_compiling" = yes; then + py_config=$host_os + case $py_config in + mingw32*) py_config=mingw32 ;; + esac + if test -f "$srcdir/Misc/cross_$py_config" ; then + AC_MSG_NOTICE([loading cross defaults from .../Misc/cross_$py_config"]) + . "$srcdir/Misc/cross_$py_config" + fi +fi + AC_MSG_CHECKING([for the Android API level]) cat > conftest.c <]],[[_beginthread(0, 0, 0);]]) + ], + [AC_MSG_RESULT([yes])], + [AC_MSG_ERROR([failed to link with nt-threads])]) +fi + +if test $with_nt_threads = yes ; then + dnl temporary default flag to avoid additional pthread checks + dnl and initilize other ac..thread flags to no + ac_cv_pthread_is_default=no + ac_cv_kthread=no + ac_cv_pthread=no + dnl ac_cv_kpthread is set to no if default is yes (see below) +else # On some compilers, pthreads are available without further options # (e.g. MacOS X). On some of these systems, the compiler will not # complain if unaccepted options are passed (e.g. gcc on Mac OS X). @@ -2613,6 +2775,8 @@ int main(void){ CC="$ac_save_cc"]) fi +fi + # If we have set a CC compiler flag for thread support then # check if it works for CXX, too. ac_cv_cxx_thread=no @@ -2633,6 +2797,10 @@ elif test "$ac_cv_pthread" = "yes" then CXX="$CXX -pthread" ac_cv_cxx_thread=yes +elif test $with_nt_threads = yes +then + dnl set to always to skip extra pthread check below + ac_cv_cxx_thread=always fi if test $ac_cv_cxx_thread = yes @@ -2667,11 +2835,11 @@ AC_DEFINE(STDC_HEADERS, 1, [Define to 1 if you have the ANSI C header files.]) # checks for header files AC_CHECK_HEADERS([ \ - alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h dlfcn.h endian.h errno.h fcntl.h grp.h \ + alloca.h asm/types.h bluetooth.h conio.h crypt.h direct.h endian.h errno.h fcntl.h grp.h \ ieeefp.h io.h langinfo.h libintl.h libutil.h linux/auxvec.h sys/auxv.h linux/limits.h linux/memfd.h \ linux/random.h linux/soundcard.h \ - linux/tipc.h linux/wait.h netdb.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ - sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ + linux/tipc.h linux/wait.h netdb.h netinet/in.h netpacket/packet.h poll.h process.h pty.h \ + setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ @@ -2679,9 +2847,24 @@ AC_CHECK_HEADERS([ \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_HEADERS([dlfcn.h]);; +esac + + AC_HEADER_DIRENT AC_HEADER_MAJOR +# If using nt threads, don't look for pthread.h or thread.h +if test "x$with_nt_threads" = xno ; then +AC_HEADER_STDC +AC_CHECK_HEADERS(pthread.h sched.h thread.h) +AC_HEADER_DIRENT +AC_HEADER_MAJOR +fi + # bluetooth/bluetooth.h has been known to not compile with -std=c99. # http://permalink.gmane.org/gmane.linux.bluez.kernel/22294 SAVE_CFLAGS=$CFLAGS @@ -2856,6 +3039,10 @@ dnl LFS does not work with Emscripten 3.1 AS_CASE([$ac_sys_system], [Emscripten], [have_largefile_support="no"] ) +dnl Activate on windows platforms (32&64-bit) where off_t(4) < fpos_t(8) +AS_CASE([$ac_sys_system], + [MINGW], [have_largefile_support="yes"] +) AS_VAR_IF([have_largefile_support], [yes], [ AC_DEFINE(HAVE_LARGEFILE_SUPPORT, 1, [Defined to enable large file support when an off_t is bigger than a long @@ -2886,6 +3073,10 @@ elif test "$ac_cv_pthread" = "yes" then CC="$CC -pthread" fi +if test $with_nt_threads = yes ; then + dnl skip check for pthread_t if NT-thread model is enabled + ac_cv_have_pthread_t=skip +else AC_CACHE_CHECK([for pthread_t], [ac_cv_have_pthread_t], [ AC_COMPILE_IFELSE([ AC_LANG_PROGRAM([[#include ]], [[pthread_t x; x = *(pthread_t*)0;]]) @@ -2917,7 +3108,7 @@ AS_VAR_IF([ac_cv_pthread_key_t_is_arithmetic_type], [yes], [ AC_DEFINE(PTHREAD_KEY_T_IS_COMPATIBLE_WITH_INT, 1, [Define if pthread_key_t is compatible with int.]) ]) - +fi CC="$ac_save_cc" AC_SUBST(OTHER_LIBTOOL_OPT) @@ -3093,6 +3284,9 @@ if test -z "$SHLIB_SUFFIX"; then CYGWIN*) SHLIB_SUFFIX=.dll;; *) SHLIB_SUFFIX=.so;; esac + case $host_os in + mingw*) SHLIB_SUFFIX=.pyd;; + esac fi AC_MSG_RESULT($SHLIB_SUFFIX) @@ -3222,6 +3416,10 @@ then CYGWIN*) LDSHARED="gcc -shared -Wl,--enable-auto-image-base" LDCXXSHARED="g++ -shared -Wl,--enable-auto-image-base";; + MINGW*) + LDSHARED='$(CC) -shared -Wl,--enable-auto-image-base' + LDCXXSHARED='$(CXX) -shared -Wl,--enable-auto-image-base' + ;; *) LDSHARED="ld";; esac fi @@ -3345,6 +3543,11 @@ then VxWorks*) LINKFORSHARED='-Wl,-export-dynamic';; esac + case $host in + *-*-mingw*) + # for https://bugs.python.org/issue40458 on MINGW + LINKFORSHARED="-Wl,--stack,2000000";; + esac fi AC_MSG_RESULT($LINKFORSHARED) @@ -3389,7 +3592,12 @@ AC_MSG_RESULT($SHLIBS) # checks for libraries AC_CHECK_LIB(sendfile, sendfile) -AC_CHECK_LIB(dl, dlopen) # Dynamic linking for SunOS/Solaris and SYSV + +case $host in + *-*-mingw*) ;; + *) AC_CHECK_LIB(dl, dlopen) ;; # Dynamic linking for SunOS/Solaris and SYSV +esac + AC_CHECK_LIB(dld, shl_load) # Dynamic linking for HP-UX @@ -3452,16 +3660,30 @@ AS_VAR_IF([have_uuid], [missing], [ AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +if test $with_nt_threads = yes ; then + dnl do not search for sem_init if NT-thread model is enabled + : +else # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux AC_SEARCH_LIBS(sem_init, pthread rt posix4) +fi # check if we need libintl for locale functions +case $host in + *-*-mingw*) + dnl Native windows build don't use libintl (see _localemodule.c). + dnl Also we don't like setup.py to add "intl" library to the list + dnl when build _locale module. + ;; + *) AC_CHECK_LIB(intl, textdomain, [AC_DEFINE(WITH_LIBINTL, 1, [Define to 1 if libintl is needed for locale functions.]) LIBS="-lintl $LIBS"]) + ;; +esac # checks for system dependent C++ extensions support case "$ac_sys_system" in @@ -3649,7 +3871,7 @@ else fi if test "$with_system_ffi" = "yes" && test -n "$PKG_CONFIG"; then - LIBFFI_INCLUDEDIR="`"$PKG_CONFIG" libffi --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ *$//'`" + LIBFFI_INCLUDEDIR="`"$PKG_CONFIG" libffi --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ .*$//'`" else LIBFFI_INCLUDEDIR="" fi @@ -3784,6 +4006,12 @@ AS_CASE([$ac_sys_system], [OSSAUDIODEV_LIBS=""] ) +dnl On MINGW, you need to link against ws2_32 and iphlpapi for sockets to work +AS_CASE([$ac_sys_system], + [MINGW], [SOCKET_LIBS="-lws2_32 -liphlpapi"], + [SOCKET_LIBS=""] +) + dnl detect sqlite3 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([LIBSQLITE3], [-sUSE_SQLITE3]) @@ -4046,6 +4274,18 @@ AS_VAR_IF([with_dbmliborder], [error], [ ]) AC_MSG_RESULT($with_dbmliborder) +case $host in + *-*-mingw*) + CFLAGS_NODIST="$CFLAGS_NODIST -D_WIN32_WINNT=0x0602";; +esac + +# Determine if windows modules should be used. +AC_SUBST(USE_WIN32_MODULE) +USE_WIN32_MODULE='#' +case $host in + *-*-mingw*) USE_WIN32_MODULE=;; +esac + # Templates for things AC_DEFINEd more than once. # For a single AC_DEFINE, no template is needed. AH_TEMPLATE(_REENTRANT, @@ -4080,6 +4320,11 @@ then CXX="$CXX -pthread" fi posix_threads=yes +elif test $with_nt_threads = yes +then + posix_threads=no + AC_DEFINE(NT_THREADS, 1, + [Define to 1 if you want to use native NT threads]) else if test ! -z "$withval" -a -d "$withval" then LDFLAGS="$LDFLAGS -L$withval" @@ -4454,11 +4699,14 @@ AC_MSG_RESULT($with_freelists) AC_MSG_CHECKING(for --with-c-locale-coercion) AC_ARG_WITH(c-locale-coercion, AS_HELP_STRING([--with-c-locale-coercion], - [enable C locale coercion to a UTF-8 based locale (default is yes)])) + [enable C locale coercion to a UTF-8 based locale (default is yes on Unix, no on Windows)])) if test -z "$with_c_locale_coercion" then - with_c_locale_coercion="yes" + case $host in + *-*-mingw*) with_c_locale_coercion="no";; + *) with_c_locale_coercion="yes";; + esac fi if test "$with_c_locale_coercion" != "no" then @@ -4559,12 +4807,36 @@ then fi ;; esac + case $host in + *-*-mingw*) + DYNLOADFILE="dynload_win.o" + extra_machdep_objs="$extra_machdep_objs PC/dl_nt.o" + CFLAGS_NODIST="$CFLAGS_NODIST -DPY3_DLLNAME='L\"$DLLLIBRARY\"'" + case $host in + i686*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-32\"'" + ;; + armv7*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm32\"'" + ;; + aarch64*) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"${VERSION}-arm64\"'" + ;; + *) + CFLAGS_NODIST="$CFLAGS_NODIST -DMS_DLL_ID='\"$VERSION\"'" + ;; + esac + ;; + esac fi AC_MSG_RESULT($DYNLOADFILE) if test "$DYNLOADFILE" != "dynload_stub.o" then + have_dynamic_loading=yes AC_DEFINE(HAVE_DYNAMIC_LOADING, 1, [Defined when any dynamic module loading is enabled.]) +else + have_dynamic_loading=no fi # MACHDEP_OBJS can be set to platform-specific object files needed by Python @@ -4584,13 +4856,22 @@ else fi # checks for library functions +if test $with_nt_threads = yes ; then + dnl GCC(mingw) 4.4+ require and use posix threads(pthreads-w32) + dnl and host may contain installed pthreads-w32. + dnl Skip checks for some functions declared in pthreads-w32 if + dnl NT-thread model is enabled. + ac_cv_func_pthread_kill=skip + ac_cv_func_sem_open=skip + ac_cv_func_sched_setscheduler=skip +fi AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock close_range confstr \ copy_file_range ctermid dup dup3 execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid getentropy geteuid getgid getgrgid getgrgid_r \ - getgrnam_r getgrouplist getgroups gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist getgroups getitimer getloadavg getlogin \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd if_nameindex initgroups kill killpg lchown linkat \ @@ -4603,7 +4884,7 @@ AC_CHECK_FUNCS([ \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ setitimer setlocale setpgid setpgrp setpriority setregid setresgid \ - setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ + setresuid setreuid setsid setuid setvbuf sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ sysconf system tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ @@ -4833,7 +5114,13 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) -AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [#include ])]) +AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 + #include +#else + #include +#endif +])]) PY_CHECK_NETDB_FUNC([hstrerror]) dnl not available in WASI yet @@ -4842,13 +5129,19 @@ PY_CHECK_NETDB_FUNC([getservbyport]) PY_CHECK_NETDB_FUNC([gethostbyname]) PY_CHECK_NETDB_FUNC([gethostbyaddr]) PY_CHECK_NETDB_FUNC([getprotobyname]) +PY_CHECK_NETDB_FUNC([gethostname]) +PY_CHECK_NETDB_FUNC([shutdown]) dnl PY_CHECK_SOCKET_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_SOCKET_FUNC], [PY_CHECK_FUNC([$1], [ +#ifdef _WIN32 +#include +#else #include #include #include #include +#endif ])]) PY_CHECK_SOCKET_FUNC([inet_aton]) @@ -4949,6 +5242,9 @@ WITH_SAVE_ENV([ ]) ]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_FUNCS(clock_gettime, [], [ AC_CHECK_LIB(rt, clock_gettime, [ LIBS="$LIBS -lrt" @@ -4969,6 +5265,8 @@ AC_CHECK_FUNCS(clock_settime, [], [ AC_DEFINE(HAVE_CLOCK_SETTIME, 1) ]) ]) + ;; +esac AC_CHECK_FUNCS(clock_nanosleep, [], [ AC_CHECK_LIB(rt, clock_nanosleep, [ @@ -5167,18 +5465,33 @@ if test $ac_cv_header_time_altzone = yes; then AC_DEFINE(HAVE_ALTZONE, 1, [Define this if your time.h defines altzone.]) fi +AC_CHECK_HEADERS([ws2tcpip.h]) AC_CACHE_CHECK([for addrinfo], [ac_cv_struct_addrinfo], -AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[struct addrinfo a]])], +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#ifdef HAVE_WS2TCPIP_H +# include +#else +# include +#endif]], + [[struct addrinfo a]])], [ac_cv_struct_addrinfo=yes], [ac_cv_struct_addrinfo=no])) if test $ac_cv_struct_addrinfo = yes; then - AC_DEFINE(HAVE_ADDRINFO, 1, [struct addrinfo (netdb.h)]) + AC_DEFINE(HAVE_ADDRINFO, 1, [struct addrinfo]) fi AC_CACHE_CHECK([for sockaddr_storage], [ac_cv_struct_sockaddr_storage], AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -# include -# include ]], [[struct sockaddr_storage s]])], +#ifdef HAVE_WS2TCPIP_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif]], + [[struct sockaddr_storage s]])], [ac_cv_struct_sockaddr_storage=yes], [ac_cv_struct_sockaddr_storage=no])) if test $ac_cv_struct_sockaddr_storage = yes; then @@ -5512,6 +5825,10 @@ dnl actually works. For FreeBSD versions <= 7.2, dnl the kernel module that provides POSIX semaphores dnl isn't loaded by default, so an attempt to call dnl sem_open results in a 'Signal 12' error. +if test $with_nt_threads = yes ; then + dnl skip posix semaphores test if NT-thread model is enabled + ac_cv_posix_semaphores_enabled=no +fi AC_CACHE_CHECK([whether POSIX semaphores are enabled], [ac_cv_posix_semaphores_enabled], AC_RUN_IFELSE([ AC_LANG_SOURCE([ @@ -5545,6 +5862,14 @@ AS_VAR_IF([ac_cv_posix_semaphores_enabled], [no], [ ]) dnl Multiprocessing check for broken sem_getvalue +if test $with_nt_threads = yes ; then + dnl Skip test if NT-thread model is enabled. + dnl NOTE the test case below fail for pthreads-w32 as: + dnl - SEM_FAILED is not defined; + dnl - sem_open is a stub; + dnl - sem_getvalue work(!). + ac_cv_broken_sem_getvalue=skip +fi AC_CACHE_CHECK([for broken sem_getvalue], [ac_cv_broken_sem_getvalue], AC_RUN_IFELSE([ AC_LANG_SOURCE([ @@ -5581,7 +5906,10 @@ AS_VAR_IF([ac_cv_broken_sem_getvalue], [yes], [ ) ]) -AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[#include ]]) +case $host in + *-*-mingw*) ;; + *) AC_CHECK_DECLS([RTLD_LAZY, RTLD_NOW, RTLD_GLOBAL, RTLD_LOCAL, RTLD_NODELETE, RTLD_NOLOAD, RTLD_DEEPBIND, RTLD_MEMBER], [], [], [[#include ]]);; +esac # determine what size digit to use for Python's longs AC_MSG_CHECKING([digit size for Python's longs]) @@ -5668,6 +5996,71 @@ esac # check for endianness AC_C_BIGENDIAN +AC_SUBST(PYD_PLATFORM_TAG) +# Special case of PYD_PLATFORM_TAG with python build with mingw. +# Python can with compiled with clang or gcc and linked +# to msvcrt or ucrt. To avoid conflicts between them +# we are selecting the extension as based on the compiler +# and the runtime they link to +# gcc + x86_64 + msvcrt = cp{version number}-x86_64 +# gcc + i686 + msvcrt = cp{version number}-i686 +# gcc + x86_64 + ucrt = cp{version number}-x86_64-ucrt +# clang + x86_64 + ucrt = cp{version number}-x86_64-clang +# clang + i686 + ucrt = cp{version number}-i686-clang + +PYD_PLATFORM_TAG="" +case $host in + *-*-mingw*) + # check if we are linking to ucrt + AC_MSG_CHECKING(whether linking to ucrt) + AC_COMPILE_IFELSE([AC_LANG_SOURCE([[ + #include + #ifndef _UCRT + #error no ucrt + #endif + int main(){ return 0; } + ]])],[linking_to_ucrt=yes],[linking_to_ucrt=no]) + AC_MSG_RESULT($linking_to_ucrt) + ;; +esac +case $host_os in + mingw*) + AC_MSG_CHECKING(PYD_PLATFORM_TAG) + case $host in + i686-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG32 + PYD_PLATFORM_TAG="mingw_i686_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_i686" + else + PYD_PLATFORM_TAG="mingw_i686_ucrt" + fi + fi + ;; + x86_64-*-mingw*) + if test -n "${cc_is_clang}"; then + # it is CLANG64 + PYD_PLATFORM_TAG="mingw_x86_64_clang" + else + if test $linking_to_ucrt = no; then + PYD_PLATFORM_TAG="mingw_x86_64" + else + PYD_PLATFORM_TAG="mingw_x86_64_ucrt" + fi + fi + ;; + aarch64-*-mingw*) + PYD_PLATFORM_TAG+="mingw_aarch64" + ;; + armv7-*-mingw*) + PYD_PLATFORM_TAG+="mingw_armv7" + ;; + esac + AC_MSG_RESULT($PYD_PLATFORM_TAG) +esac + # ABI version string for Python extension modules. This appears between the # periods in shared library file names, e.g. foo..so. It is calculated # from the following attributes which affect the ABI of this Python build (in @@ -5700,7 +6093,12 @@ if test "$Py_DEBUG" = 'true' -a "$with_trace_refs" != "yes"; then fi AC_SUBST(EXT_SUFFIX) -EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +VERSION_NO_DOTS=$(echo $LDVERSION | tr -d .) +if test -n "${PYD_PLATFORM_TAG}"; then + EXT_SUFFIX=".cp${VERSION_NO_DOTS}-${PYD_PLATFORM_TAG}${SHLIB_SUFFIX}" +else + EXT_SUFFIX=.${SOABI}${SHLIB_SUFFIX} +fi AC_MSG_CHECKING(LDVERSION) LDVERSION='$(VERSION)$(ABIFLAGS)' @@ -5708,7 +6106,7 @@ AC_MSG_RESULT($LDVERSION) # On Android and Cygwin the shared libraries must be linked with libpython. AC_SUBST(LIBPYTHON) -if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin"; then +if test -n "$ANDROID_API_LEVEL" -o "$MACHDEP" = "cygwin" -o "$MACHDEP" = "win32"; then LIBPYTHON="-lpython${VERSION}${ABIFLAGS}" else LIBPYTHON='' @@ -6077,11 +6475,16 @@ then [Define if you have struct stat.st_mtimensec]) fi +if test -n "$PKG_CONFIG"; then + NCURSESW_INCLUDEDIR="`"$PKG_CONFIG" ncursesw --cflags-only-I 2>/dev/null | sed -e 's/^-I//;s/ .*$//'`" +else + NCURSESW_INCLUDEDIR="" +fi +AC_SUBST(NCURSESW_INCLUDEDIR) + # first curses header check ac_save_cppflags="$CPPFLAGS" -if test "$cross_compiling" = no; then - CPPFLAGS="$CPPFLAGS -I/usr/include/ncursesw" -fi +CPPFLAGS="$CPPFLAGS -I$NCURSESW_INCLUDEDIR" AC_CHECK_HEADERS(curses.h ncurses.h) @@ -6247,7 +6650,10 @@ fi AC_CHECK_TYPE(socklen_t,, AC_DEFINE(socklen_t,int, - [Define to `int' if does not define.]),[ + [Define to `int' if or does not define.]),[ +#ifdef HAVE_WS2TCPIP_H +#include +#endif #ifdef HAVE_SYS_TYPES_H #include #endif @@ -6337,6 +6743,27 @@ do THREADHEADERS="$THREADHEADERS \$(srcdir)/$h" done +case $host in + *-*-mingw*) + dnl Required for windows builds as Objects/exceptions.c require + dnl "errmap.h" from $srcdir/PC. + dnl Note we cannot use BASECPPFLAGS as autogenerated pyconfig.h + dnl has to be before customized located in ../PC. + dnl (-I. at end is workaround for setup.py logic) + CPPFLAGS="-I\$(srcdir)/PC $CPPFLAGS -I." + ;; +esac + +dnl Python interpreter main program for frozen scripts +AC_SUBST(PYTHON_OBJS_FROZENMAIN) +PYTHON_OBJS_FROZENMAIN="Python/frozenmain.o" +case $host in + *-*-mingw*) + dnl 'PC/frozen_dllmain.c' - not yet + PYTHON_OBJS_FROZENMAIN= + ;; +esac + AC_SUBST(SRCDIRS) SRCDIRS="\ Modules \ @@ -6358,6 +6785,10 @@ SRCDIRS="\ Python \ Python/frozen_modules \ Python/deepfreeze" +case $host in + *-*-mingw*) SRCDIRS="$SRCDIRS PC";; +esac + AC_MSG_CHECKING(for build directories) for dir in $SRCDIRS; do if test ! -d $dir; then @@ -6366,6 +6797,38 @@ for dir in $SRCDIRS; do done AC_MSG_RESULT(done) +# For mingw build need additional library for linking +case $host in + *-*-mingw*) + LIBS="$LIBS -lversion -lshlwapi -lpathcch -lbcrypt" + AC_PROG_AWK + if test "$AWK" = "gawk"; then + awk_extra_flag="--non-decimal-data" + fi + AC_MSG_CHECKING([FIELD3]) + FIELD3=$($AWK $awk_extra_flag '\ + /^#define PY_RELEASE_LEVEL_/ {levels[$2]=$3} \ + /^#define PY_MICRO_VERSION[[:space:]]+/ {micro=$3} \ + /^#define PY_RELEASE_LEVEL[[:space:]]+/ {level=levels[$3]} \ + /^#define PY_RELEASE_SERIAL[[:space:]]+/ {serial=$3} \ + END {print micro * 1000 + level * 10 + serial}' \ + $srcdir/Include/patchlevel.h + ) + + AC_MSG_RESULT([${FIELD3}]) + RCFLAGS="$RCFLAGS -DFIELD3=$FIELD3 -O COFF" + + case $host in + i686*) RCFLAGS="$RCFLAGS --target=pe-i386" ;; + x86_64*) RCFLAGS="$RCFLAGS --target=pe-x86-64" ;; + *) ;; + esac + ;; + *) + ;; +esac +AC_SUBST(RCFLAGS) + # Availability of -O2: AC_CACHE_CHECK([for -O2], [ac_cv_compile_o2], [ saved_cflags="$CFLAGS" @@ -6975,7 +7438,6 @@ PY_STDLIB_MOD_SIMPLE([_json]) PY_STDLIB_MOD_SIMPLE([_lsprof]) PY_STDLIB_MOD_SIMPLE([_opcode]) PY_STDLIB_MOD_SIMPLE([_pickle]) -PY_STDLIB_MOD_SIMPLE([_posixsubprocess]) PY_STDLIB_MOD_SIMPLE([_queue]) PY_STDLIB_MOD_SIMPLE([_random]) PY_STDLIB_MOD_SIMPLE([select]) @@ -6986,7 +7448,7 @@ PY_STDLIB_MOD_SIMPLE([_zoneinfo]) dnl multiprocessing modules PY_STDLIB_MOD([_multiprocessing], - [], [test "$ac_cv_func_sem_unlink" = "yes"], + [], [test "$ac_cv_func_sem_unlink" = "yes" -o "$MACHDEP" = "win32"], [-I\$(srcdir)/Modules/_multiprocessing]) PY_STDLIB_MOD([_posixshmem], [], [test "$have_posix_shmem" = "yes"], @@ -7006,11 +7468,15 @@ PY_STDLIB_MOD([fcntl], [], [test "$ac_cv_header_sys_ioctl_h" = "yes" -a "$ac_cv_header_fcntl_h" = "yes"], [], [$FCNTL_LIBS]) PY_STDLIB_MOD([mmap], - [], [test "$ac_cv_header_sys_mman_h" = "yes" -a "$ac_cv_header_sys_stat_h" = "yes"]) + [], m4_flatten([test "$ac_cv_header_sys_mman_h" = "yes" + -a "$ac_cv_header_sys_stat_h" = "yes" + -o "$MACHDEP" = "win32"])) PY_STDLIB_MOD([_socket], [], m4_flatten([test "$ac_cv_header_sys_socket_h" = "yes" -a "$ac_cv_header_sys_types_h" = "yes" - -a "$ac_cv_header_netinet_in_h" = "yes"])) + -a "$ac_cv_header_netinet_in_h" = "yes" + -o "$MACHDEP" = "win32"]), + [], [$SOCKET_LIBS]) dnl platform specific extensions PY_STDLIB_MOD([grp], [], [test "$ac_cv_func_getgrgid" = yes -o "$ac_cv_func_getgrgid_r" = yes]) @@ -7025,6 +7491,7 @@ PY_STDLIB_MOD([_scproxy], PY_STDLIB_MOD([spwd], [], [test "$ac_cv_func_getspent" = yes -o "$ac_cv_func_getspnam" = yes]) PY_STDLIB_MOD([syslog], [], [test "$ac_cv_header_syslog_h" = yes]) PY_STDLIB_MOD([termios], [], [test "$ac_cv_header_termios_h" = yes]) +PY_STDLIB_MOD([_posixsubprocess], [], [test "$MACHDEP" != "win32"]) dnl _elementtree loads libexpat via CAPI hook in pyexpat PY_STDLIB_MOD([pyexpat], [], [], [$LIBEXPAT_CFLAGS], [$LIBEXPAT_LDFLAGS]) @@ -7087,25 +7554,35 @@ PY_STDLIB_MOD([_lzma], [], [test "$have_liblzma" = yes], dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) + [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS -lws2_32]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) +dnl windows specific modules +PY_STDLIB_MOD([msvcrt], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_winapi], [test "$MACHDEP" = "win32"]) +PY_STDLIB_MOD([_msi], [test "$MACHDEP" = "win32"], [], [], + [-lmsi -lcabinet -lrpcrt4]) +PY_STDLIB_MOD([winsound], [test "$MACHDEP" = "win32"], [], [], + [-lwinmm]) +PY_STDLIB_MOD([_overlapped], [test "$MACHDEP" = "win32"], [], [], + [-lws2_32]) + dnl test modules PY_STDLIB_MOD([_testcapi], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testclinic], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_testinternalcapi], [test "$TEST_MODULES" = yes], [], [-DPY3_DLLNAME="\"$DLLLIBRARY\""], []) PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) -PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes], [], [-lm]) +PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_dynamic_loading" = yes], [], [-lm]) dnl Limited API template modules. dnl The limited C API is not compatible with the Py_TRACE_REFS macro. dnl Emscripten does not support shared libraries yet. -PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$ac_cv_func_dlopen" = yes]) +PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) +PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"], [test "$have_dynamic_loading" = yes]) # substitute multiline block, must come after last PY_STDLIB_MOD() AC_SUBST([MODULE_BLOCK]) diff --git a/mingw_ignorefile.txt b/mingw_ignorefile.txt new file mode 100644 index 00000000000000..54093f1a0512b7 --- /dev/null +++ b/mingw_ignorefile.txt @@ -0,0 +1,42 @@ +ctypes.test.test_loading.LoaderTest.test_load_dll_with_flags +distutils.tests.test_bdist_dumb.BuildDumbTestCase.test_simple_built +distutils.tests.test_cygwinccompiler.CygwinCCompilerTestCase.test_get_versions +distutils.tests.test_util.UtilTestCase.test_change_root +test.datetimetester.TestLocalTimeDisambiguation_Fast.* +test.datetimetester.TestLocalTimeDisambiguation_Pure.* +test.test_cmath.CMathTests.test_specific_values +test.test_cmd_line_script.CmdLineTest.test_consistent_sys_path_for_direct_execution +test.test_compileall.CommandLineTestsNoSourceEpoch.* +test.test_compileall.CommandLineTestsWithSourceEpoch.* +test.test_compileall.CompileallTestsWithoutSourceEpoch.* +test.test_compileall.CompileallTestsWithSourceEpoch.* +test.test_import.ImportTests.test_dll_dependency_import +test.test_math.MathTests.* +test.test_ntpath.NtCommonTest.test_import +test.test_os.StatAttributeTests.test_stat_block_device +test.test_os.TestScandir.test_attributes +test.test_os.UtimeTests.test_large_time +test.test_platform.PlatformTest.test_architecture_via_symlink +test.test_regrtest.ProgramsTestCase.test_pcbuild_rt +test.test_regrtest.ProgramsTestCase.test_tools_buildbot_test +test.test_site._pthFileTests.* +test.test_site.HelperFunctionsTests.* +test.test_site.StartupImportTests.* +test.test_ssl.* +test.test_strptime.CalculationTests.* +test.test_strptime.StrptimeTests.test_weekday +test.test_strptime.TimeRETests.test_compile +test.test_tools.test_i18n.Test_pygettext.test_POT_Creation_Date +test.test_venv.BasicTest.* +test.test_venv.EnsurePipTest.* +test.test_sysconfig.TestSysConfig.test_user_similar +test.test_tcl.TclTest.testLoadWithUNC +# flaky +test.test__xxsubinterpreters.* +test.test_asyncio.test_subprocess.SubprocessProactorTests.test_stdin_broken_pipe +test.test_asynchat.TestAsynchat.test_line_terminator2 +test.test_asyncgen.AsyncGenAsyncioTest.test_async_gen_asyncio_gc_aclose_09 +test.test_concurrent_futures.ThreadPoolShutdownTest.test_interpreter_shutdown +test.test_asynchat.TestNotConnected.test_disallow_negative_terminator +test.test_logging.SysLogHandlerTest.* +test.test_logging.IPv6SysLogHandlerTest.* diff --git a/mingw_smoketests.py b/mingw_smoketests.py new file mode 100644 index 00000000000000..ca1f652caeec27 --- /dev/null +++ b/mingw_smoketests.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +# Copyright 2017 Christoph Reiter +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +"""The goal of this test suite is collect tests for update regressions +and to test msys2 related modifications like for path handling. +Feel free to extend. +""" + +import os +import unittest +import sysconfig + +if os.environ.get("MSYSTEM", ""): + SEP = "/" +else: + SEP = "\\" + +if sysconfig.is_python_build(): + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + +_UCRT = sysconfig.get_platform() not in ('mingw_x86_64', 'mingw_i686') + + +class Tests(unittest.TestCase): + + def test_zoneinfo(self): + # https://github.com/msys2-contrib/cpython-mingw/issues/32 + import zoneinfo + self.assertTrue(any(os.path.exists(p) for p in zoneinfo.TZPATH)) + zoneinfo.ZoneInfo("America/Sao_Paulo") + + def test_userdir_path_sep(self): + # Make sure os.path and pathlib use the same path separators + from unittest import mock + from os.path import expanduser + from pathlib import Path + + profiles = ["C:\\foo", "C:/foo"] + for profile in profiles: + with mock.patch.dict(os.environ, {"USERPROFILE": profile}): + self.assertEqual(expanduser("~"), os.path.normpath(expanduser("~"))) + self.assertEqual(str(Path("~").expanduser()), expanduser("~")) + self.assertEqual(str(Path.home()), expanduser("~")) + + def test_sysconfig_schemes(self): + # https://github.com/msys2/MINGW-packages/issues/9319 + import sysconfig + from distutils.dist import Distribution + from distutils.command.install import install + + names = ['scripts', 'purelib', 'platlib', 'data', 'include'] + for scheme in ["nt", "nt_user"]: + for name in names: + c = install(Distribution({"name": "foobar"})) + c.user = (scheme == "nt_user") + c.finalize_options() + if name == "include": + dist_path = os.path.dirname(getattr(c, "install_" + "headers")) + else: + dist_path = getattr(c, "install_" + name) + sys_path = sysconfig.get_path(name, scheme) + self.assertEqual(dist_path, sys_path, (scheme, name)) + + def test_ctypes_find_library(self): + from ctypes.util import find_library + from ctypes import cdll + self.assertTrue(cdll.msvcrt) + if _UCRT: + self.assertIsNone(find_library('c')) + else: + self.assertEqual(find_library('c'), 'msvcrt.dll') + + def test_ctypes_dlopen(self): + import ctypes + import sys + self.assertEqual(ctypes.RTLD_GLOBAL, 0) + self.assertEqual(ctypes.RTLD_GLOBAL, ctypes.RTLD_LOCAL) + self.assertFalse(hasattr(sys, 'setdlopenflags')) + self.assertFalse(hasattr(sys, 'getdlopenflags')) + self.assertFalse([n for n in dir(os) if n.startswith("RTLD_")]) + + def test_time_no_unix_stuff(self): + import time + self.assertFalse([n for n in dir(time) if n.startswith("clock_")]) + self.assertFalse([n for n in dir(time) if n.startswith("CLOCK_")]) + self.assertFalse([n for n in dir(time) if n.startswith("pthread_")]) + self.assertFalse(hasattr(time, 'tzset')) + + def test_strftime(self): + import time + with self.assertRaises(ValueError): + time.strftime('%Y', (12345,) + (0,) * 8) + + def test_sep(self): + self.assertEqual(os.sep, SEP) + + def test_module_file_path(self): + import asyncio + import zlib + self.assertEqual(zlib.__file__, os.path.normpath(zlib.__file__)) + self.assertEqual(asyncio.__file__, os.path.normpath(asyncio.__file__)) + + def test_importlib_frozen_path_sep(self): + import importlib._bootstrap_external + self.assertEqual(importlib._bootstrap_external.path_sep, SEP) + + def test_os_commonpath(self): + self.assertEqual( + os.path.commonpath( + [os.path.join("C:", os.sep, "foo", "bar"), + os.path.join("C:", os.sep, "foo")]), + os.path.join("C:", os.sep, "foo")) + + def test_pathlib(self): + import pathlib + + p = pathlib.Path("foo") / pathlib.Path("foo") + self.assertEqual(str(p), os.path.normpath(p)) + + def test_modules_import(self): + import sqlite3 + import ssl + import ctypes + import curses + + def test_c_modules_import(self): + import _decimal + + def test_socket_inet_ntop(self): + import socket + self.assertTrue(hasattr(socket, "inet_ntop")) + + def test_socket_inet_pton(self): + import socket + self.assertTrue(hasattr(socket, "inet_pton")) + + def test_multiprocessing_queue(self): + from multiprocessing import Queue + Queue(0) + + #def test_socket_timout_normal_error(self): + # import urllib.request + # from urllib.error import URLError + + # try: + # urllib.request.urlopen( + # 'http://localhost', timeout=0.0001).close() + # except URLError: + # pass + + def test_threads(self): + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(1) as pool: + for res in pool.map(lambda *x: None, range(10000)): + pass + + def test_sysconfig(self): + import sysconfig + # This should be able to execute without exceptions + sysconfig.get_config_vars() + + def test_sqlite_enable_load_extension(self): + # Make sure --enable-loadable-sqlite-extensions is used + import sqlite3 + self.assertTrue(sqlite3.Connection.enable_load_extension) + + def test_venv_creation(self): + import tempfile + import venv + import subprocess + import shutil + with tempfile.TemporaryDirectory() as tmp: + builder = venv.EnvBuilder() + builder.create(tmp) + assert os.path.exists(os.path.join(tmp, "bin", "activate")) + assert os.path.exists(os.path.join(tmp, "bin", "python.exe")) + assert os.path.exists(os.path.join(tmp, "bin", "python3.exe")) + subprocess.check_call([shutil.which("bash.exe"), os.path.join(tmp, "bin", "activate")]) + + # This will not work in in-tree build + if not sysconfig.is_python_build(): + op = subprocess.check_output( + [ + os.path.join(tmp, "bin", "python.exe"), + "-c", + "print('Hello World')" + ], + cwd=tmp, + ) + assert op.decode().strip() == "Hello World" + + def test_has_mktime(self): + from time import mktime, gmtime + mktime(gmtime()) + + def test_platform_things(self): + import sys + import sysconfig + import platform + import importlib.machinery + self.assertEqual(sys.implementation.name, "cpython") + self.assertEqual(sys.platform, "win32") + self.assertTrue(sysconfig.get_platform().startswith("mingw")) + self.assertTrue(sysconfig.get_config_var('SOABI').startswith("cpython-")) + ext_suffix = sysconfig.get_config_var('EXT_SUFFIX') + self.assertTrue(ext_suffix.endswith(".pyd")) + self.assertTrue("mingw" in ext_suffix) + self.assertEqual(sysconfig.get_config_var('SHLIB_SUFFIX'), ".pyd") + ext_suffixes = importlib.machinery.EXTENSION_SUFFIXES + self.assertTrue(ext_suffix in ext_suffixes) + self.assertTrue(".pyd" in ext_suffixes) + if sysconfig.get_platform().startswith('mingw_i686'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-32') + elif sysconfig.get_platform().startswith('mingw_aarch64'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm64') + elif sysconfig.get_platform().startswith('mingw_armv7'): + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2])) + '-arm32') + else: + self.assertEqual(sys.winver, ".".join(map(str, sys.version_info[:2]))) + self.assertEqual(platform.python_implementation(), "CPython") + self.assertEqual(platform.system(), "Windows") + self.assertTrue(isinstance(sys.api_version, int) and sys.api_version > 0) + + def test_sys_getpath(self): + # everything sourced from getpath.py + import sys + + def assertNormpath(path): + self.assertEqual(path, os.path.normpath(path)) + + assertNormpath(sys.executable) + assertNormpath(sys._base_executable) + assertNormpath(sys.prefix) + assertNormpath(sys.base_prefix) + assertNormpath(sys.exec_prefix) + assertNormpath(sys.base_exec_prefix) + assertNormpath(sys.platlibdir) + assertNormpath(sys._stdlib_dir) + for p in sys.path: + assertNormpath(p) + + def test_site(self): + import site + + self.assertEqual(len(site.getsitepackages()), 1) + + def test_c_ext_build(self): + import tempfile + import sys + import subprocess + import textwrap + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmppro: + subprocess.check_call([sys.executable, "-m", "ensurepip", "--user"]) + with Path(tmppro, "setup.py").open("w") as f: + f.write( + textwrap.dedent( + """\ + from setuptools import setup, Extension + + setup( + name='cwrapper', + version='1.0', + ext_modules=[ + Extension( + 'cwrapper', + sources=['cwrapper.c']), + ], + ) + """ + ) + ) + with Path(tmppro, "cwrapper.c").open("w") as f: + f.write( + textwrap.dedent( + """\ + #include + static PyObject * + helloworld(PyObject *self, PyObject *args) + { + printf("Hello World\\n"); + Py_RETURN_NONE; + } + static PyMethodDef + myMethods[] = { + { "helloworld", helloworld, METH_NOARGS, "Prints Hello World" }, + { NULL, NULL, 0, NULL } + }; + static struct PyModuleDef cwrapper = { + PyModuleDef_HEAD_INIT, + "cwrapper", + "Test Module", + -1, + myMethods + }; + + PyMODINIT_FUNC + PyInit_cwrapper(void) + { + return PyModule_Create(&cwrapper); + } + """ + ) + ) + subprocess.check_call( + [sys.executable, "-c", "import struct"], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "wheel", + ], + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + tmppro, + ], + ) + subprocess.check_call( + [sys.executable, "-c", "import cwrapper"], + ) + + + +def suite(): + return unittest.TestLoader().loadTestsFromName(__name__) + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/pyconfig.h.in b/pyconfig.h.in index 0536047f573ce6..c8e3b39246603f 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -60,7 +60,7 @@ /* Define to 1 if you have the `acosh' function. */ #undef HAVE_ACOSH -/* struct addrinfo (netdb.h) */ +/* struct addrinfo */ #undef HAVE_ADDRINFO /* Define to 1 if you have the `alarm' function. */ @@ -1524,6 +1524,9 @@ /* Define if mvwdelch in curses.h is an expression. */ #undef MVWDELCH_IS_EXPRESSION +/* Define to 1 if you want to use native NT threads */ +#undef NT_THREADS + /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT @@ -1839,7 +1842,7 @@ /* Define to `unsigned int' if does not define. */ #undef size_t -/* Define to `int' if does not define. */ +/* Define to `int' if or does not define. */ #undef socklen_t /* Define to `int' if doesn't define. */ diff --git a/setup.py b/setup.py index 4f122b62e0e708..946ee91a41b74b 100644 --- a/setup.py +++ b/setup.py @@ -77,9 +77,24 @@ def get_platform(): return sys.platform +# On MSYS, os.system needs to be wrapped with sh.exe +# as otherwise all the io redirection will fail. +# Arguably, this could happen inside the real os.system +# rather than this monkey patch. +if sys.platform == "win32" and os.environ.get("MSYSTEM", ""): + os_system = os.system + def msys_system(command): + command_in_sh = 'sh.exe -c "%s"' % command.replace("\\", "\\\\") + return os_system(command_in_sh) + os.system = msys_system + + # set PYTHONLEGACYWINDOWSDLLLOADING to 1 to load DLLs from PATH + # This is needed while building core extensions of Python + os.environ["PYTHONLEGACYWINDOWSDLLLOADING"] = "1" + CROSS_COMPILING = ("_PYTHON_HOST_PLATFORM" in os.environ) HOST_PLATFORM = get_platform() -MS_WINDOWS = (HOST_PLATFORM == 'win32') +MS_WINDOWS = (HOST_PLATFORM == 'win32' or HOST_PLATFORM == 'mingw') CYGWIN = (HOST_PLATFORM == 'cygwin') MACOS = (HOST_PLATFORM == 'darwin') AIX = (HOST_PLATFORM.startswith('aix')) @@ -687,7 +702,7 @@ def check_extension_import(self, ext): def add_multiarch_paths(self): # Debian/Ubuntu multiarch support. # https://wiki.ubuntu.com/MultiarchSpec - tmpfile = os.path.join(self.build_temp, 'multiarch') + tmpfile = os.path.join(self.build_temp, 'multiarch').replace('\\','/') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) ret = run_command( @@ -712,7 +727,7 @@ def add_multiarch_paths(self): opt = '' if CROSS_COMPILING: opt = '-t' + sysconfig.get_config_var('HOST_GNU_TYPE') - tmpfile = os.path.join(self.build_temp, 'multiarch') + tmpfile = os.path.join(self.build_temp, 'multiarch').replace('\\','/') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) ret = run_command( @@ -774,7 +789,7 @@ def add_search_path(line): pass def add_cross_compiling_paths(self): - tmpfile = os.path.join(self.build_temp, 'ccpaths') + tmpfile = os.path.join(self.build_temp, 'ccpaths').replace('\\','/') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) # bpo-38472: With a German locale, GCC returns "gcc-Version 9.1.0 @@ -796,14 +811,25 @@ def add_cross_compiling_paths(self): elif line.startswith("End of search list"): in_incdirs = False elif (is_gcc or is_clang) and line.startswith("LIBRARY_PATH"): - for d in line.strip().split("=")[1].split(":"): + for d in line.strip().split("=")[1].split(os.pathsep): d = os.path.normpath(d) - if '/gcc/' not in d: + if '/gcc/' not in d and '/clang/' not in d: add_dir_to_list(self.compiler.library_dirs, d) elif (is_gcc or is_clang) and in_incdirs and '/gcc/' not in line and '/clang/' not in line: add_dir_to_list(self.compiler.include_dirs, line.strip()) + if is_clang: + ret = run_command('%s -print-search-dirs >%s' % (CC, tmpfile)) + if ret == 0: + with open(tmpfile) as fp: + for line in fp.readlines(): + if line.startswith("libraries:"): + for d in line.strip().split("=")[1].split(os.pathsep): + d = os.path.normpath(d) + if '/gcc/' not in d and '/clang/' not in d: + add_dir_to_list(self.compiler.library_dirs, + d) finally: os.unlink(tmpfile) @@ -849,10 +875,10 @@ def configure_compiler(self): if not CROSS_COMPILING: add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + self.add_multiarch_paths() # only change this for cross builds for 3.3, issues on Mageia if CROSS_COMPILING: self.add_cross_compiling_paths() - self.add_multiarch_paths() self.add_ldflags_cppflags() def init_inc_lib_dirs(self): @@ -863,10 +889,8 @@ def init_inc_lib_dirs(self): # (PYTHONFRAMEWORK is set) to avoid # linking problems when # building a framework with different architectures than # the one that is currently installed (issue #7473) - add_dir_to_list(self.compiler.library_dirs, - sysconfig.get_config_var("LIBDIR")) - add_dir_to_list(self.compiler.include_dirs, - sysconfig.get_config_var("INCLUDEDIR")) + self.compiler.library_dirs.append(sysconfig.get_config_var("LIBDIR")) + self.compiler.include_dirs.append(sysconfig.get_config_var("INCLUDEDIR")) system_lib_dirs = ['/lib64', '/usr/lib64', '/lib', '/usr/lib'] system_include_dirs = ['/usr/include'] @@ -898,7 +922,7 @@ def init_inc_lib_dirs(self): if HOST_PLATFORM == 'hp-ux11': self.lib_dirs += ['/usr/lib/hpux64', '/usr/lib/hpux32'] - if MACOS: + if MACOS or MS_WINDOWS: # This should work on any unixy platform ;-) # If the user has bothered specifying additional -I and -L flags # in OPT and LDFLAGS we might as well use them here. @@ -970,11 +994,15 @@ def detect_simple_extensions(self): # grp(3) self.addext(Extension('grp', ['grpmodule.c'])) - self.addext(Extension('_socket', ['socketmodule.c'])) + self.addext(Extension( + '_socket', ['socketmodule.c'], + libraries=(['ws2_32', 'iphlpapi'] if MS_WINDOWS else None))) self.addext(Extension('spwd', ['spwdmodule.c'])) # select(2); not on ancient System V - self.addext(Extension('select', ['selectmodule.c'])) + self.addext(Extension( + 'select', ['selectmodule.c'], + libraries=(['ws2_32'] if MS_WINDOWS else None))) # Memory-mapped files (also works on Win32). self.addext(Extension('mmap', ['mmapmodule.c'])) @@ -1033,12 +1061,15 @@ def detect_test_extensions(self): ['_xxtestfuzz/_xxtestfuzz.c', '_xxtestfuzz/fuzzer.c'] )) + if MS_WINDOWS: + self.add(Extension('_testconsole', ['../PC/_testconsole.c'])) + def detect_readline_curses(self): # readline readline_termcap_library = "" curses_library = "" # Cannot use os.popen here in py3k. - tmpfile = os.path.join(self.build_temp, 'readline_termcap_lib') + tmpfile = os.path.join(self.build_temp, 'readline_termcap_lib').replace('\\','/') if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) # Determine if readline is already linked against curses or tinfo. @@ -1101,6 +1132,8 @@ def detect_readline_curses(self): # readline package if find_file('readline/rlconf.h', self.inc_dirs, []) is None: do_readline = False + if MS_WINDOWS: + do_readline = False if do_readline: readline_libs = [readline_lib] if readline_termcap_library: @@ -1124,8 +1157,7 @@ def detect_readline_curses(self): panel_library = 'panel' if curses_library == 'ncursesw': curses_defines.append(('HAVE_NCURSESW', '1')) - if not CROSS_COMPILING: - curses_includes.append('/usr/include/ncursesw') + curses_includes.append(sysconfig.get_config_var("NCURSESW_INCLUDEDIR")) # Bug 1464056: If _curses.so links with ncursesw, # _curses_panel.so must link with panelw. panel_library = 'panelw' @@ -1209,7 +1241,7 @@ def detect_dbm_gdbm(self): if dbm_args: dbm_order = [arg.split('=')[-1] for arg in dbm_args][-1].split(":") else: - dbm_order = "gdbm:ndbm:bdb".split(":") + dbm_order = [] dbmext = None for cand in dbm_order: if cand == "ndbm": @@ -1282,6 +1314,19 @@ def detect_platform_specific_exts(self): # macOS-only, needs SystemConfiguration and CoreFoundation framework self.addext(Extension('_scproxy', ['_scproxy.c'])) + # Windows-only modules + if MS_WINDOWS: + srcdir = sysconfig.get_config_var('srcdir') + pc_srcdir = os.path.abspath(os.path.join(srcdir, 'PC')) + + self.addext(Extension('_msi', + [os.path.join(pc_srcdir, '_msi.c')])) + + self.addext(Extension('winsound', + [os.path.join(pc_srcdir, 'winsound.c')])) + + self.addext(Extension('_overlapped', ['overlapped.c'])) + def detect_compress_exts(self): # Andrew Kuchling's zlib module. self.addext(Extension('zlib', ['zlibmodule.c'])) @@ -1329,9 +1374,10 @@ def detect_multiprocessing(self): if ( sysconfig.get_config_var('HAVE_SEM_OPEN') and not sysconfig.get_config_var('POSIX_SEMAPHORES_NOT_ENABLED') - ): + ) or MS_WINDOWS: multiprocessing_srcs.append('_multiprocessing/semaphore.c') - self.addext(Extension('_multiprocessing', multiprocessing_srcs)) + self.addext(Extension('_multiprocessing', multiprocessing_srcs, + libraries=(['ws2_32'] if MS_WINDOWS else None), include_dirs=["Modules/_multiprocessing"])) self.addext(Extension('_posixshmem', ['_multiprocessing/posixshmem.c'])) def detect_uuid(self): @@ -1414,11 +1460,16 @@ def detect_ctypes(self): include_dirs=include_dirs, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, - libraries=[], + libraries=(['ole32', 'oleaut32', 'uuid'] if MS_WINDOWS else []), + export_symbols=( + ['DllGetClassObject PRIVATE', 'DllCanUnloadNow PRIVATE'] + if MS_WINDOWS else None + ), sources=sources) self.add(ext) # function my_sqrt() needs libm for sqrt() - self.addext(Extension('_ctypes_test', ['_ctypes/_ctypes_test.c'])) + self.addext(Extension('_ctypes_test', ['_ctypes/_ctypes_test.c'], + libraries=(['oleaut32'] if MS_WINDOWS else []))) ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") ffi_lib = None @@ -1568,7 +1619,7 @@ def copy_scripts(self): else: newfilename = filename + minoronly log.info(f'renaming {filename} to {newfilename}') - os.rename(filename, newfilename) + os.replace(filename, newfilename) newoutfiles.append(newfilename) if filename in updated_files: newupdated_files.append(newfilename)