Skip to content

Commit

Permalink
gh-91321: Fix compatibility with C++ older than C++11 (#93784) (#93802)
Browse files Browse the repository at this point in the history
* Fix the compatibility of the Python C API with C++ older than C++11.
* _Py_NULL is only defined as nullptr on C++11 and newer.

(cherry picked from commit 4caf5c2)

* test_cppext now builds the C++ extension with setuptools.
* Add @test.support.requires_venv_with_pip.

(cherry picked from commit ca0cc9c)
  • Loading branch information
vstinner authored Jun 14, 2022
1 parent 871b1dc commit ef591cf
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 88 deletions.
7 changes: 5 additions & 2 deletions Include/pyport.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ extern "C++" {
inline type _Py_CAST_impl(int ptr) {
return reinterpret_cast<type>(ptr);
}
#if __cplusplus >= 201103
template <typename type>
inline type _Py_CAST_impl(std::nullptr_t) {
return static_cast<type>(nullptr);
}
#endif

template <typename type, typename expr_type>
inline type _Py_CAST_impl(expr_type *expr) {
Expand Down Expand Up @@ -70,8 +72,9 @@ extern "C++" {
#endif

// Static inline functions should use _Py_NULL rather than using directly NULL
// to prevent C++ compiler warnings. In C++, _Py_NULL uses nullptr.
#ifdef __cplusplus
// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as
// nullptr.
#if defined(__cplusplus) && __cplusplus >= 201103
# define _Py_NULL nullptr
#else
# define _Py_NULL NULL
Expand Down
46 changes: 30 additions & 16 deletions Lib/test/_testcppext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@

#include "Python.h"

#if __cplusplus >= 201103
# define NAME _testcpp11ext
#else
# define NAME _testcpp03ext
#endif

PyDoc_STRVAR(_testcppext_add_doc,
"add(x, y)\n"
"\n"
Expand All @@ -16,7 +22,7 @@ _testcppext_add(PyObject *Py_UNUSED(module), PyObject *args)
{
long i, j;
if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) {
return nullptr;
return _Py_NULL;
}
long res = i + j;
return PyLong_FromLong(res);
Expand Down Expand Up @@ -47,8 +53,8 @@ static PyObject *
test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *obj = Py_BuildValue("(ii)", 1, 2);
if (obj == nullptr) {
return nullptr;
if (obj == _Py_NULL) {
return _Py_NULL;
}
Py_ssize_t refcnt = Py_REFCNT(obj);
assert(refcnt >= 1);
Expand Down Expand Up @@ -77,9 +83,11 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
// gh-93442: Pass 0 as NULL for PyObject*
Py_XINCREF(0);
Py_XDECREF(0);
// ensure that nullptr works too
#if _cplusplus >= 201103
// Test nullptr passed as PyObject*
Py_XINCREF(nullptr);
Py_XDECREF(nullptr);
#endif

Py_DECREF(obj);
Py_RETURN_NONE;
Expand All @@ -90,16 +98,16 @@ static PyObject *
test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))
{
PyObject *str = PyUnicode_FromString("abc");
if (str == nullptr) {
return nullptr;
if (str == _Py_NULL) {
return _Py_NULL;
}

assert(PyUnicode_Check(str));
assert(PyUnicode_GET_LENGTH(str) == 3);

// gh-92800: test PyUnicode_READ()
const void* data = PyUnicode_DATA(str);
assert(data != nullptr);
assert(data != _Py_NULL);
int kind = PyUnicode_KIND(str);
assert(kind == PyUnicode_1BYTE_KIND);
assert(PyUnicode_READ(kind, data, 0) == 'a');
Expand All @@ -118,9 +126,9 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args))

static PyMethodDef _testcppext_methods[] = {
{"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc},
{"test_api_casts", test_api_casts, METH_NOARGS, nullptr},
{"test_unicode", test_unicode, METH_NOARGS, nullptr},
{nullptr, nullptr, 0, nullptr} /* sentinel */
{"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL},
{"test_unicode", test_unicode, METH_NOARGS, _Py_NULL},
{_Py_NULL, _Py_NULL, 0, _Py_NULL} /* sentinel */
};


Expand All @@ -135,26 +143,32 @@ _testcppext_exec(PyObject *module)

static PyModuleDef_Slot _testcppext_slots[] = {
{Py_mod_exec, reinterpret_cast<void*>(_testcppext_exec)},
{0, nullptr}
{0, _Py_NULL}
};


PyDoc_STRVAR(_testcppext_doc, "C++ test extension.");

#define _STR(NAME) #NAME
#define STR(NAME) _STR(NAME)

static struct PyModuleDef _testcppext_module = {
PyModuleDef_HEAD_INIT, // m_base
"_testcppext", // m_name
STR(NAME), // m_name
_testcppext_doc, // m_doc
0, // m_size
_testcppext_methods, // m_methods
_testcppext_slots, // m_slots
nullptr, // m_traverse
nullptr, // m_clear
nullptr, // m_free
_Py_NULL, // m_traverse
_Py_NULL, // m_clear
_Py_NULL, // m_free
};

#define _FUNC_NAME(NAME) PyInit_ ## NAME
#define FUNC_NAME(NAME) _FUNC_NAME(NAME)

PyMODINIT_FUNC
PyInit__testcppext(void)
FUNC_NAME(NAME)(void)
{
return PyModuleDef_Init(&_testcppext_module);
}
51 changes: 51 additions & 0 deletions Lib/test/setup_testcppext.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# gh-91321: Build a basic C++ test extension to check that the Python C API is
# compatible with C++ and does not emit C++ compiler warnings.
import sys
from test import support

from setuptools import setup, Extension


MS_WINDOWS = (sys.platform == 'win32')


SOURCE = support.findfile('_testcppext.cpp')
if not MS_WINDOWS:
# C++ compiler flags for GCC and clang
CPPFLAGS = [
# gh-91321: The purpose of _testcppext extension is to check that building
# a C++ extension using the Python C API does not emit C++ compiler
# warnings
'-Werror',
# Warn on old-style cast (C cast) like: (PyObject*)op
'-Wold-style-cast',
# Warn when using NULL rather than _Py_NULL in static inline functions
'-Wzero-as-null-pointer-constant',
]
else:
# Don't pass any compiler flag to MSVC
CPPFLAGS = []


def main():
cppflags = list(CPPFLAGS)
if '-std=c++03' in sys.argv:
sys.argv.remove('-std=c++03')
std = 'c++03'
name = '_testcpp03ext'
else:
# Python currently targets C++11
std = 'c++11'
name = '_testcpp11ext'

cppflags = [*CPPFLAGS, f'-std={std}']
cpp_ext = Extension(
name,
sources=[SOURCE],
language='c++',
extra_compile_args=cppflags)
setup(name=name, ext_modules=[cpp_ext])


if __name__ == "__main__":
main()
17 changes: 17 additions & 0 deletions Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2196,3 +2196,20 @@ def clear_ignored_deprecations(*tokens: object) -> None:
if warnings.filters != new_filters:
warnings.filters[:] = new_filters
warnings._filters_mutated()


# Skip a test if venv with pip is known to not work.
def requires_venv_with_pip():
# ensurepip requires zlib to open ZIP archives (.whl binary wheel packages)
try:
import zlib
except ImportError:
return unittest.skipIf(True, "venv: ensurepip requires zlib")

# bpo-26610: pip/pep425tags.py requires ctypes.
# gh-92820: setuptools/windows_support.py uses ctypes (setuptools 58.1).
try:
import ctypes
except ImportError:
ctypes = None
return unittest.skipUnless(ctypes, 'venv: pip requires ctypes')
114 changes: 48 additions & 66 deletions Lib/test/test_cppext.py
Original file line number Diff line number Diff line change
@@ -1,91 +1,73 @@
# gh-91321: Build a basic C++ test extension to check that the Python C API is
# compatible with C++ and does not emit C++ compiler warnings.
import contextlib
import os
import os.path
import sys
import unittest
import warnings
import subprocess
from test import support
from test.support import os_helper

with warnings.catch_warnings():
warnings.simplefilter('ignore', DeprecationWarning)
from distutils.core import setup, Extension
import distutils.sysconfig


MS_WINDOWS = (sys.platform == 'win32')


SOURCE = support.findfile('_testcppext.cpp')
if not MS_WINDOWS:
# C++ compiler flags for GCC and clang
CPPFLAGS = [
# Python currently targets C++11
'-std=c++11',
# gh-91321: The purpose of _testcppext extension is to check that building
# a C++ extension using the Python C API does not emit C++ compiler
# warnings
'-Werror',
# Warn on old-style cast (C cast) like: (PyObject*)op
'-Wold-style-cast',
# Warn when using NULL rather than _Py_NULL in static inline functions
'-Wzero-as-null-pointer-constant',
]
else:
# Don't pass any compiler flag to MSVC
CPPFLAGS = []
SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py')


@support.requires_subprocess()
class TestCPPExt(unittest.TestCase):
def build(self):
cpp_ext = Extension(
'_testcppext',
sources=[SOURCE],
language='c++',
extra_compile_args=CPPFLAGS)
capture_stdout = (not support.verbose)
def test_build_cpp11(self):
self.check_build(False)

try:
try:
if capture_stdout:
stdout = support.captured_stdout()
else:
print()
stdout = contextlib.nullcontext()
with (stdout,
support.swap_attr(sys, 'argv', ['setup.py', 'build_ext', '--verbose'])):
setup(name="_testcppext", ext_modules=[cpp_ext])
return
except:
if capture_stdout:
# Show output on error
print()
print(stdout.getvalue())
raise
except SystemExit:
self.fail("Build failed")
def test_build_cpp03(self):
self.check_build(True)

# With MSVC, the linker fails with: cannot open file 'python311.lib'
# https://github.com/python/cpython/pull/32175#issuecomment-1111175897
@unittest.skipIf(MS_WINDOWS, 'test fails on Windows')
def test_build(self):
# save/restore os.environ
def restore_env(old_env):
os.environ.clear()
os.environ.update(old_env)
self.addCleanup(restore_env, dict(os.environ))

def restore_sysconfig_vars(old_config_vars):
distutils.sysconfig._config_vars.clear()
distutils.sysconfig._config_vars.update(old_config_vars)
self.addCleanup(restore_sysconfig_vars,
dict(distutils.sysconfig._config_vars))

# the test uses venv+pip: skip if it's not available
@support.requires_venv_with_pip()
def check_build(self, std_cpp03):
# Build in a temporary directory
with os_helper.temp_cwd():
self.build()
self._check_build(std_cpp03)

def _check_build(self, std_cpp03):
venv_dir = 'env'
verbose = support.verbose

# Create virtual environment to get setuptools
cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir]
if verbose:
print()
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True)

# Get the Python executable of the venv
python_exe = 'python'
if sys.executable.endswith('.exe'):
python_exe += '.exe'
if MS_WINDOWS:
python = os.path.join(venv_dir, 'Scripts', python_exe)
else:
python = os.path.join(venv_dir, 'bin', python_exe)

# Build the C++ extension
cmd = [python, '-X', 'dev',
SETUP_TESTCPPEXT, 'build_ext', '--verbose']
if std_cpp03:
cmd.append('-std=c++03')
if verbose:
print('Run:', ' '.join(cmd))
subprocess.run(cmd, check=True)
else:
proc = subprocess.run(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True)
if proc.returncode:
print(proc.stdout, end='')
self.fail(f"Build failed with exit code {proc.returncode}")


if __name__ == "__main__":
Expand Down
7 changes: 3 additions & 4 deletions Lib/test/test_venv.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@
import tempfile
from test.support import (captured_stdout, captured_stderr, requires_zlib,
skip_if_broken_multiprocessing_synchronize, verbose,
requires_subprocess, is_emscripten, is_wasi)
requires_subprocess, is_emscripten, is_wasi,
requires_venv_with_pip)
from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree)
import unittest
import venv
Expand Down Expand Up @@ -619,9 +620,7 @@ def do_test_with_pip(self, system_site_packages):
if not system_site_packages:
self.assert_pip_not_installed()

# Issue #26610: pip/pep425tags.py requires ctypes
@unittest.skipUnless(ctypes, 'pip requires ctypes')
@requires_zlib()
@requires_venv_with_pip()
def test_with_pip(self):
self.do_test_with_pip(False)
self.do_test_with_pip(True)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Fix the compatibility of the Python C API with C++ older than C++11. Patch by
Victor Stinner.

0 comments on commit ef591cf

Please sign in to comment.