From ef591cf8e3725e74489f4c19ca85b87cf6886852 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jun 2022 16:05:14 +0200 Subject: [PATCH] gh-91321: Fix compatibility with C++ older than C++11 (#93784) (#93802) * Fix the compatibility of the Python C API with C++ older than C++11. * _Py_NULL is only defined as nullptr on C++11 and newer. (cherry picked from commit 4caf5c2753f1aa28d6f4bc1aa377975fd2a62331) * test_cppext now builds the C++ extension with setuptools. * Add @test.support.requires_venv_with_pip. (cherry picked from commit ca0cc9c433830e14714a5cc93fb4e7254da3dd76) --- Include/pyport.h | 7 +- Lib/test/_testcppext.cpp | 46 ++++--- Lib/test/setup_testcppext.py | 51 ++++++++ Lib/test/support/__init__.py | 17 +++ Lib/test/test_cppext.py | 114 ++++++++---------- Lib/test/test_venv.py | 7 +- ...2-06-13-21-37-31.gh-issue-91321.DgJFvS.rst | 2 + 7 files changed, 156 insertions(+), 88 deletions(-) create mode 100644 Lib/test/setup_testcppext.py create mode 100644 Misc/NEWS.d/next/C API/2022-06-13-21-37-31.gh-issue-91321.DgJFvS.rst diff --git a/Include/pyport.h b/Include/pyport.h index a78e290931fffe..59f225fc8fa5d7 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -36,10 +36,12 @@ extern "C++" { inline type _Py_CAST_impl(int ptr) { return reinterpret_cast(ptr); } +#if __cplusplus >= 201103 template inline type _Py_CAST_impl(std::nullptr_t) { return static_cast(nullptr); } +#endif template inline type _Py_CAST_impl(expr_type *expr) { @@ -70,8 +72,9 @@ extern "C++" { #endif // Static inline functions should use _Py_NULL rather than using directly NULL -// to prevent C++ compiler warnings. In C++, _Py_NULL uses nullptr. -#ifdef __cplusplus +// to prevent C++ compiler warnings. On C++11 and newer, _Py_NULL is defined as +// nullptr. +#if defined(__cplusplus) && __cplusplus >= 201103 # define _Py_NULL nullptr #else # define _Py_NULL NULL diff --git a/Lib/test/_testcppext.cpp b/Lib/test/_testcppext.cpp index 5e3d76b7b20723..b6d35407a61edc 100644 --- a/Lib/test/_testcppext.cpp +++ b/Lib/test/_testcppext.cpp @@ -6,6 +6,12 @@ #include "Python.h" +#if __cplusplus >= 201103 +# define NAME _testcpp11ext +#else +# define NAME _testcpp03ext +#endif + PyDoc_STRVAR(_testcppext_add_doc, "add(x, y)\n" "\n" @@ -16,7 +22,7 @@ _testcppext_add(PyObject *Py_UNUSED(module), PyObject *args) { long i, j; if (!PyArg_ParseTuple(args, "ll:foo", &i, &j)) { - return nullptr; + return _Py_NULL; } long res = i + j; return PyLong_FromLong(res); @@ -47,8 +53,8 @@ static PyObject * test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { PyObject *obj = Py_BuildValue("(ii)", 1, 2); - if (obj == nullptr) { - return nullptr; + if (obj == _Py_NULL) { + return _Py_NULL; } Py_ssize_t refcnt = Py_REFCNT(obj); assert(refcnt >= 1); @@ -77,9 +83,11 @@ test_api_casts(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) // gh-93442: Pass 0 as NULL for PyObject* Py_XINCREF(0); Py_XDECREF(0); - // ensure that nullptr works too +#if _cplusplus >= 201103 + // Test nullptr passed as PyObject* Py_XINCREF(nullptr); Py_XDECREF(nullptr); +#endif Py_DECREF(obj); Py_RETURN_NONE; @@ -90,8 +98,8 @@ static PyObject * test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { PyObject *str = PyUnicode_FromString("abc"); - if (str == nullptr) { - return nullptr; + if (str == _Py_NULL) { + return _Py_NULL; } assert(PyUnicode_Check(str)); @@ -99,7 +107,7 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) // gh-92800: test PyUnicode_READ() const void* data = PyUnicode_DATA(str); - assert(data != nullptr); + assert(data != _Py_NULL); int kind = PyUnicode_KIND(str); assert(kind == PyUnicode_1BYTE_KIND); assert(PyUnicode_READ(kind, data, 0) == 'a'); @@ -118,9 +126,9 @@ test_unicode(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) static PyMethodDef _testcppext_methods[] = { {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc}, - {"test_api_casts", test_api_casts, METH_NOARGS, nullptr}, - {"test_unicode", test_unicode, METH_NOARGS, nullptr}, - {nullptr, nullptr, 0, nullptr} /* sentinel */ + {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL}, + {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL}, + {_Py_NULL, _Py_NULL, 0, _Py_NULL} /* sentinel */ }; @@ -135,26 +143,32 @@ _testcppext_exec(PyObject *module) static PyModuleDef_Slot _testcppext_slots[] = { {Py_mod_exec, reinterpret_cast(_testcppext_exec)}, - {0, nullptr} + {0, _Py_NULL} }; PyDoc_STRVAR(_testcppext_doc, "C++ test extension."); +#define _STR(NAME) #NAME +#define STR(NAME) _STR(NAME) + static struct PyModuleDef _testcppext_module = { PyModuleDef_HEAD_INIT, // m_base - "_testcppext", // m_name + STR(NAME), // m_name _testcppext_doc, // m_doc 0, // m_size _testcppext_methods, // m_methods _testcppext_slots, // m_slots - nullptr, // m_traverse - nullptr, // m_clear - nullptr, // m_free + _Py_NULL, // m_traverse + _Py_NULL, // m_clear + _Py_NULL, // m_free }; +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + PyMODINIT_FUNC -PyInit__testcppext(void) +FUNC_NAME(NAME)(void) { return PyModuleDef_Init(&_testcppext_module); } diff --git a/Lib/test/setup_testcppext.py b/Lib/test/setup_testcppext.py new file mode 100644 index 00000000000000..a288dbdc57bded --- /dev/null +++ b/Lib/test/setup_testcppext.py @@ -0,0 +1,51 @@ +# gh-91321: Build a basic C++ test extension to check that the Python C API is +# compatible with C++ and does not emit C++ compiler warnings. +import sys +from test import support + +from setuptools import setup, Extension + + +MS_WINDOWS = (sys.platform == 'win32') + + +SOURCE = support.findfile('_testcppext.cpp') +if not MS_WINDOWS: + # C++ compiler flags for GCC and clang + CPPFLAGS = [ + # gh-91321: The purpose of _testcppext extension is to check that building + # a C++ extension using the Python C API does not emit C++ compiler + # warnings + '-Werror', + # Warn on old-style cast (C cast) like: (PyObject*)op + '-Wold-style-cast', + # Warn when using NULL rather than _Py_NULL in static inline functions + '-Wzero-as-null-pointer-constant', + ] +else: + # Don't pass any compiler flag to MSVC + CPPFLAGS = [] + + +def main(): + cppflags = list(CPPFLAGS) + if '-std=c++03' in sys.argv: + sys.argv.remove('-std=c++03') + std = 'c++03' + name = '_testcpp03ext' + else: + # Python currently targets C++11 + std = 'c++11' + name = '_testcpp11ext' + + cppflags = [*CPPFLAGS, f'-std={std}'] + cpp_ext = Extension( + name, + sources=[SOURCE], + language='c++', + extra_compile_args=cppflags) + setup(name=name, ext_modules=[cpp_ext]) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 35c4efb01af27f..2aec065d4da457 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2196,3 +2196,20 @@ def clear_ignored_deprecations(*tokens: object) -> None: if warnings.filters != new_filters: warnings.filters[:] = new_filters warnings._filters_mutated() + + +# Skip a test if venv with pip is known to not work. +def requires_venv_with_pip(): + # ensurepip requires zlib to open ZIP archives (.whl binary wheel packages) + try: + import zlib + except ImportError: + return unittest.skipIf(True, "venv: ensurepip requires zlib") + + # bpo-26610: pip/pep425tags.py requires ctypes. + # gh-92820: setuptools/windows_support.py uses ctypes (setuptools 58.1). + try: + import ctypes + except ImportError: + ctypes = None + return unittest.skipUnless(ctypes, 'venv: pip requires ctypes') diff --git a/Lib/test/test_cppext.py b/Lib/test/test_cppext.py index 337cb08f8c9d8c..8673911ecfae5d 100644 --- a/Lib/test/test_cppext.py +++ b/Lib/test/test_cppext.py @@ -1,91 +1,73 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. -import contextlib -import os +import os.path import sys import unittest -import warnings +import subprocess from test import support from test.support import os_helper -with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - from distutils.core import setup, Extension - import distutils.sysconfig - MS_WINDOWS = (sys.platform == 'win32') -SOURCE = support.findfile('_testcppext.cpp') -if not MS_WINDOWS: - # C++ compiler flags for GCC and clang - CPPFLAGS = [ - # Python currently targets C++11 - '-std=c++11', - # gh-91321: The purpose of _testcppext extension is to check that building - # a C++ extension using the Python C API does not emit C++ compiler - # warnings - '-Werror', - # Warn on old-style cast (C cast) like: (PyObject*)op - '-Wold-style-cast', - # Warn when using NULL rather than _Py_NULL in static inline functions - '-Wzero-as-null-pointer-constant', - ] -else: - # Don't pass any compiler flag to MSVC - CPPFLAGS = [] +SETUP_TESTCPPEXT = support.findfile('setup_testcppext.py') @support.requires_subprocess() class TestCPPExt(unittest.TestCase): - def build(self): - cpp_ext = Extension( - '_testcppext', - sources=[SOURCE], - language='c++', - extra_compile_args=CPPFLAGS) - capture_stdout = (not support.verbose) + def test_build_cpp11(self): + self.check_build(False) - try: - try: - if capture_stdout: - stdout = support.captured_stdout() - else: - print() - stdout = contextlib.nullcontext() - with (stdout, - support.swap_attr(sys, 'argv', ['setup.py', 'build_ext', '--verbose'])): - setup(name="_testcppext", ext_modules=[cpp_ext]) - return - except: - if capture_stdout: - # Show output on error - print() - print(stdout.getvalue()) - raise - except SystemExit: - self.fail("Build failed") + def test_build_cpp03(self): + self.check_build(True) # With MSVC, the linker fails with: cannot open file 'python311.lib' # https://github.com/python/cpython/pull/32175#issuecomment-1111175897 @unittest.skipIf(MS_WINDOWS, 'test fails on Windows') - def test_build(self): - # save/restore os.environ - def restore_env(old_env): - os.environ.clear() - os.environ.update(old_env) - self.addCleanup(restore_env, dict(os.environ)) - - def restore_sysconfig_vars(old_config_vars): - distutils.sysconfig._config_vars.clear() - distutils.sysconfig._config_vars.update(old_config_vars) - self.addCleanup(restore_sysconfig_vars, - dict(distutils.sysconfig._config_vars)) - + # the test uses venv+pip: skip if it's not available + @support.requires_venv_with_pip() + def check_build(self, std_cpp03): # Build in a temporary directory with os_helper.temp_cwd(): - self.build() + self._check_build(std_cpp03) + + def _check_build(self, std_cpp03): + venv_dir = 'env' + verbose = support.verbose + + # Create virtual environment to get setuptools + cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] + if verbose: + print() + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + + # Get the Python executable of the venv + python_exe = 'python' + if sys.executable.endswith('.exe'): + python_exe += '.exe' + if MS_WINDOWS: + python = os.path.join(venv_dir, 'Scripts', python_exe) + else: + python = os.path.join(venv_dir, 'bin', python_exe) + + # Build the C++ extension + cmd = [python, '-X', 'dev', + SETUP_TESTCPPEXT, 'build_ext', '--verbose'] + if std_cpp03: + cmd.append('-std=c++03') + if verbose: + print('Run:', ' '.join(cmd)) + subprocess.run(cmd, check=True) + else: + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True) + if proc.returncode: + print(proc.stdout, end='') + self.fail(f"Build failed with exit code {proc.returncode}") if __name__ == "__main__": diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 199160e4d0e72a..37b61a780cb07b 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -17,7 +17,8 @@ import tempfile from test.support import (captured_stdout, captured_stderr, requires_zlib, skip_if_broken_multiprocessing_synchronize, verbose, - requires_subprocess, is_emscripten, is_wasi) + requires_subprocess, is_emscripten, is_wasi, + requires_venv_with_pip) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) import unittest import venv @@ -619,9 +620,7 @@ def do_test_with_pip(self, system_site_packages): if not system_site_packages: self.assert_pip_not_installed() - # Issue #26610: pip/pep425tags.py requires ctypes - @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @requires_venv_with_pip() def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) diff --git a/Misc/NEWS.d/next/C API/2022-06-13-21-37-31.gh-issue-91321.DgJFvS.rst b/Misc/NEWS.d/next/C API/2022-06-13-21-37-31.gh-issue-91321.DgJFvS.rst new file mode 100644 index 00000000000000..57c39bc8d83c87 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-06-13-21-37-31.gh-issue-91321.DgJFvS.rst @@ -0,0 +1,2 @@ +Fix the compatibility of the Python C API with C++ older than C++11. Patch by +Victor Stinner.