diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 49101c18756860..90e45f65e298e1 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -289,7 +289,6 @@ PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); PyAPI_FUNC(void) _Py_BreakPoint(void); PyAPI_FUNC(void) _PyObject_Dump(PyObject *); -PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); PyAPI_FUNC(int) _PyObject_IsAbstract(PyObject *); PyAPI_FUNC(PyObject *) _PyObject_GetAttrId(PyObject *, _Py_Identifier *); @@ -377,12 +376,6 @@ PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); #endif -PyAPI_FUNC(void) -_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, - size_t sizeof_block); -PyAPI_FUNC(void) -_PyObject_DebugTypeStats(FILE *out); - /* Define a pair of assertion macros: _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT(). @@ -431,21 +424,6 @@ PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed( int line, const char *function); -/* Check if an object is consistent. For example, ensure that the reference - counter is greater than or equal to 1, and ensure that ob_type is not NULL. - - Call _PyObject_AssertFailed() if the object is inconsistent. - - If check_content is zero, only check header fields: reduce the overhead. - - The function always return 1. The return value is just here to be able to - write: - - assert(_PyObject_CheckConsistency(obj, 1)); */ -PyAPI_FUNC(int) _PyObject_CheckConsistency( - PyObject *op, - int check_content); - /* Trashcan mechanism, thanks to Christian Tismer. diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index a9c996557430ad..661820c894dc0e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,6 +14,27 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime +/* Check if an object is consistent. For example, ensure that the reference + counter is greater than or equal to 1, and ensure that ob_type is not NULL. + + Call _PyObject_AssertFailed() if the object is inconsistent. + + If check_content is zero, only check header fields: reduce the overhead. + + The function always return 1. The return value is just here to be able to + write: + + assert(_PyObject_CheckConsistency(obj, 1)); */ +extern int _PyObject_CheckConsistency(PyObject *op, int check_content); + +extern void _PyDebugAllocatorStats(FILE *out, const char *block_name, + int num_blocks, size_t sizeof_block); + +extern void _PyObject_DebugTypeStats(FILE *out); + +// Export for shared _testinternalcapi extension +PyAPI_DATA(int) _PyObject_IsFreed(PyObject *); + /* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid designated initializer conflicts in C++20. If we use the deinition in object.h, we will be mixing designated and non-designated initializers in diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index a9ff410cb93ab8..527000875b7241 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -8,8 +8,10 @@ from test.support.script_helper import assert_python_failure, assert_python_ok -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi and _testinternalcapi extensions are not +# available. _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') @requires_subprocess() class PyMemDebugTests(unittest.TestCase): @@ -84,16 +86,13 @@ def test_pyobject_malloc_without_gil(self): def check_pyobject_is_freed(self, func_name): code = textwrap.dedent(f''' - import gc, os, sys, _testcapi + import gc, os, sys, _testinternalcapi # Disable the GC to avoid crash on GC collection gc.disable() - try: - _testcapi.{func_name}() - # Exit immediately to avoid a crash while deallocating - # the invalid object - os._exit(0) - except _testcapi.error: - os._exit(1) + _testinternalcapi.{func_name}() + # Exit immediately to avoid a crash while deallocating + # the invalid object + os._exit(0) ''') assert_python_ok( '-c', code, diff --git a/Modules/_testcapi/mem.c b/Modules/_testcapi/mem.c index 979b3a4b2b1af6..0b2b1cd31fb0a1 100644 --- a/Modules/_testcapi/mem.c +++ b/Modules/_testcapi/mem.c @@ -526,75 +526,6 @@ pymem_malloc_without_gil(PyObject *self, PyObject *args) Py_RETURN_NONE; } -static PyObject * -test_pyobject_is_freed(const char *test_name, PyObject *op) -{ - if (!_PyObject_IsFreed(op)) { - PyErr_SetString(PyExc_AssertionError, - "object is not seen as freed"); - return NULL; - } - Py_RETURN_NONE; -} - -static PyObject * -check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyObject *op = NULL; - return test_pyobject_is_freed("check_pyobject_null_is_freed", op); -} - - -static PyObject * -check_pyobject_uninitialized_is_freed(PyObject *self, - PyObject *Py_UNUSED(args)) -{ - PyObject *op = (PyObject *)PyObject_Malloc(sizeof(PyObject)); - if (op == NULL) { - return NULL; - } - /* Initialize reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* object fields like ob_type are uninitialized! */ - return test_pyobject_is_freed("check_pyobject_uninitialized_is_freed", op); -} - - -static PyObject * -check_pyobject_forbidden_bytes_is_freed(PyObject *self, - PyObject *Py_UNUSED(args)) -{ - /* Allocate an incomplete PyObject structure: truncate 'ob_type' field */ - PyObject *op = (PyObject *)PyObject_Malloc(offsetof(PyObject, ob_type)); - if (op == NULL) { - return NULL; - } - /* Initialize reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* ob_type field is after the memory block: part of "forbidden bytes" - when using debug hooks on memory allocators! */ - return test_pyobject_is_freed("check_pyobject_forbidden_bytes_is_freed", op); -} - - -static PyObject * -check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) -{ - /* This test would fail if run with the address sanitizer */ -#ifdef _Py_ADDRESS_SANITIZER - Py_RETURN_NONE; -#else - PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); - if (op == NULL) { - return NULL; - } - Py_TYPE(op)->tp_dealloc(op); - /* Reset reference count to avoid early crash in ceval or GC */ - Py_SET_REFCNT(op, 1); - /* object memory is freed! */ - return test_pyobject_is_freed("check_pyobject_freed_is_freed", op); -#endif -} // Tracemalloc tests static PyObject * @@ -656,12 +587,6 @@ tracemalloc_untrack(PyObject *self, PyObject *args) } static PyMethodDef test_methods[] = { - {"check_pyobject_forbidden_bytes_is_freed", - check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, - {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, - {"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS}, - {"check_pyobject_uninitialized_is_freed", - check_pyobject_uninitialized_is_freed, METH_NOARGS}, {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, {"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS}, diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 26cdb4371ca24c..8ba8f8ecadec7a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -6,6 +6,7 @@ #undef NDEBUG #include "Python.h" +#include "pycore_object.h" // _PyObject_IsFreed() // Used for clone_with_conv_f1 and clone_with_conv_v2 diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ecc2721a4988f9..e2be9a130217d6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -10,7 +10,6 @@ #undef NDEBUG #include "Python.h" -#include "frameobject.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_bytesobject.h" // _PyBytes_Find() @@ -23,9 +22,12 @@ #include "pycore_initconfig.h" // _Py_GetConfigsAsDict() #include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() #include "pycore_interp_id.h" // _PyInterpreterID_LookUp() +#include "pycore_object.h" // _PyObject_IsFreed() #include "pycore_pathconfig.h" // _PyPathConfig_ClearGlobal() #include "pycore_pyerrors.h" // _Py_UTF8_Edit_Cost() #include "pycore_pystate.h" // _PyThreadState_GET() + +#include "frameobject.h" #include "osdefs.h" // MAXPATHLEN #include "clinic/_testinternalcapi.c.h" @@ -1446,6 +1448,77 @@ test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject * +test_pyobject_is_freed(const char *test_name, PyObject *op) +{ + if (!_PyObject_IsFreed(op)) { + PyErr_SetString(PyExc_AssertionError, + "object is not seen as freed"); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *op = NULL; + return test_pyobject_is_freed("check_pyobject_null_is_freed", op); +} + + +static PyObject * +check_pyobject_uninitialized_is_freed(PyObject *self, + PyObject *Py_UNUSED(args)) +{ + PyObject *op = (PyObject *)PyObject_Malloc(sizeof(PyObject)); + if (op == NULL) { + return NULL; + } + /* Initialize reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* object fields like ob_type are uninitialized! */ + return test_pyobject_is_freed("check_pyobject_uninitialized_is_freed", op); +} + + +static PyObject * +check_pyobject_forbidden_bytes_is_freed(PyObject *self, + PyObject *Py_UNUSED(args)) +{ + /* Allocate an incomplete PyObject structure: truncate 'ob_type' field */ + PyObject *op = (PyObject *)PyObject_Malloc(offsetof(PyObject, ob_type)); + if (op == NULL) { + return NULL; + } + /* Initialize reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* ob_type field is after the memory block: part of "forbidden bytes" + when using debug hooks on memory allocators! */ + return test_pyobject_is_freed("check_pyobject_forbidden_bytes_is_freed", op); +} + + +static PyObject * +check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args)) +{ + /* This test would fail if run with the address sanitizer */ +#ifdef _Py_ADDRESS_SANITIZER + Py_RETURN_NONE; +#else + PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type); + if (op == NULL) { + return NULL; + } + Py_TYPE(op)->tp_dealloc(op); + /* Reset reference count to avoid early crash in ceval or GC */ + Py_SET_REFCNT(op, 1); + /* object memory is freed! */ + return test_pyobject_is_freed("check_pyobject_freed_is_freed", op); +#endif +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -1502,6 +1575,12 @@ static PyMethodDef module_functions[] = { {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, {"test_atexit", test_atexit, METH_NOARGS}, + {"check_pyobject_forbidden_bytes_is_freed", + check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, + {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, + {"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS}, + {"check_pyobject_uninitialized_is_freed", + check_pyobject_uninitialized_is_freed, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 26e2dc31e3664a..41ae1fca90b468 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -118,7 +118,7 @@ As a consequence of this, split keys have a maximum size of 16. #include "pycore_code.h" // stats #include "pycore_dict.h" // PyDictKeysObject #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_setobject.h" // _PySet_NextEntry() diff --git a/Objects/floatobject.c b/Objects/floatobject.c index fa55481f09dec0..6a0c2e033e3e9a 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -10,7 +10,7 @@ #include "pycore_interp.h" // _PyInterpreterState.float_state #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_Init() +#include "pycore_object.h" // _PyObject_Init(), _PyDebugAllocatorStats() #include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_structseq.h" // _PyStructSequence_FiniBuiltin() diff --git a/Objects/listobject.c b/Objects/listobject.c index 144ede6351e03c..c0da9dd916851a 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -6,7 +6,7 @@ #include "pycore_list.h" // struct _Py_list_state, _PyListIterObject #include "pycore_long.h" // _PyLong_DigitCount #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _PyDebugAllocatorStats() #include "pycore_tuple.h" // _PyTuple_FromArray() #include diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index eb68d7c030d293..7d552ff57c8f1e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_code.h" // stats +#include "pycore_object.h" // _PyDebugAllocatorStats() definition #include "pycore_obmalloc.h" #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pymem.h" diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index e85af2b75e4738..c3ff40fdb14c60 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -6,7 +6,7 @@ #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_modsupport.h" // _PyArg_NoKwnames() -#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError(), _PyDebugAllocatorStats() /*[clinic input] class tuple "PyTupleObject *" "&PyTuple_Type" diff --git a/Python/sysmodule.c b/Python/sysmodule.c index fea3f61ee01762..fb033a6b5634ac 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -22,7 +22,7 @@ Data members: #include "pycore_long.h" // _PY_LONG_MAX_STR_DIGITS_THRESHOLD #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_namespace.h" // _PyNamespace_New() -#include "pycore_object.h" // _PyObject_IS_GC() +#include "pycore_object.h" // _PyObject_IS_GC(), _PyObject_DebugTypeStats() #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _PyErr_WriteUnraisableDefaultHook()