diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 32bd19d968b917..9f693141da10c6 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -18,11 +18,9 @@ extern "C" { #define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 -#define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 -#define _Py_MAX_MANAGED_STATIC_TYPES \ - (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) + all the managed static types will fit (for all builds). + That includes all static builtin types and managed extension types. */ +#define _Py_MAX_MANAGED_STATIC_TYPES 210 struct _types_runtime_state { /* Used to set PyTypeObject.tp_version_tag for core static types. */ @@ -30,9 +28,20 @@ struct _types_runtime_state { // because of static types. unsigned int next_version_tag; + /* We track every managed static type. Each one is assigned an + * index when it is first initialized using _PyStaticType_InitBuiltin() + * or _PyStaticType_InitForExtension(). It keeps that index for the + * duration of the process. The same index is used when accessing + * the global array of type state, as well as the per-interpreter + * array. */ struct { + PyMutex mutex; + size_t num_builtins; + size_t num_types; + size_t next_index; struct { PyTypeObject *type; + int isbuiltin; int64_t interp_count; } types[_Py_MAX_MANAGED_STATIC_TYPES]; } managed_static; @@ -56,25 +65,14 @@ struct type_cache { struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; }; -typedef struct { - PyTypeObject *type; - int isbuiltin; - int readying; - int ready; - // XXX tp_dict can probably be statically allocated, - // instead of dynamically and stored on the interpreter. - PyObject *tp_dict; - PyObject *tp_subclasses; - /* We never clean up weakrefs for static builtin types since - they will effectively never get triggered. However, there - are also some diagnostic uses for the list of weakrefs, - so we still keep it. */ - PyObject *tp_weaklist; -} managed_static_type_state; + +typedef struct managed_static_type_state managed_static_type_state; #define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ struct types_state { + PyMutex mutex; + /* Used to set PyTypeObject.tp_version_tag. It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1, where all those lower numbers are used for core static types. */ @@ -119,16 +117,23 @@ struct types_state { builtin type. Once initialization is over for a subinterpreter, the value will be the same as for all other interpreters. */ struct { - size_t num_initialized; - managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]; - } builtins; - /* We apply a similar strategy for managed extension modules. */ - struct { - size_t num_initialized; - size_t next_index; - managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; - } for_extensions; - PyMutex mutex; + size_t num_builtins; + size_t num_types; + struct managed_static_type_state { + PyTypeObject *type; + int readying; + int ready; + // XXX tp_dict can probably be statically allocated, + // instead of dynamically and stored on the interpreter. + PyObject *tp_dict; + PyObject *tp_subclasses; + /* We never clean up weakrefs for static builtin types since + they will effectively never get triggered. However, there + are also some diagnostic uses for the list of weakrefs, + so we still keep it. */ + PyObject *tp_weaklist; + } types[_Py_MAX_MANAGED_STATIC_TYPES]; + } managed_static; // Borrowed references to type objects whose // tp_version_tag % TYPE_VERSION_CACHE_SIZE diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 958f42430c80a2..82cc5d233fd88b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -127,36 +127,58 @@ type_from_ref(PyObject *ref) } -/* helpers for for static builtin types */ +/* helpers for for managed static types */ -#ifndef NDEBUG static inline int managed_static_type_index_is_set(PyTypeObject *self) { return self->tp_subclasses != NULL; } -#endif static inline size_t managed_static_type_index_get(PyTypeObject *self) { assert(managed_static_type_index_is_set(self)); /* We store a 1-based index so 0 can mean "not initialized". */ - return (size_t)self->tp_subclasses - 1; + size_t index = (size_t)self->tp_subclasses - 1; + assert(index < _Py_MAX_MANAGED_STATIC_TYPES); + return index; } -static inline void -managed_static_type_index_set(PyTypeObject *self, size_t index) +static size_t +managed_static_type_index_init(PyTypeObject *self) { - assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + assert(!managed_static_type_index_is_set(self)); + + PyMutex_Lock(&_PyRuntime.types.managed_static.mutex); + size_t index = _PyRuntime.types.managed_static.next_index; + _PyRuntime.types.managed_static.next_index += 1; + PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex); + assert(index < _Py_MAX_MANAGED_STATIC_TYPES); + /* We store a 1-based index so 0 can mean "not initialized". */ self->tp_subclasses = (PyObject *)(index + 1); + return index; +} + +static size_t +managed_static_type_index_reinit(PyTypeObject *self) +{ + size_t index = managed_static_type_index_get(self); + PyMutex_Lock(&_PyRuntime.types.managed_static.mutex); + if (index >= _PyRuntime.types.managed_static.next_index) { + _PyRuntime.types.managed_static.next_index = index + 1; + } + PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex); + return index; } static inline void -managed_static_type_index_clear(PyTypeObject *self) +managed_static_type_index_clear(PyTypeObject *Py_UNUSED(self)) { - self->tp_subclasses = NULL; + /* We actually leave the index set on the type (in tp_subclasses). + That way it will occupy the same array entry if re-initialized, + rather than advancing next_index. */ } static PyTypeObject * @@ -164,39 +186,31 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index, int64_t *p_interp_count) { assert(interp->runtime == &_PyRuntime); - assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); - - size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - int64_t interp_count = - _PyRuntime.types.managed_static.types[full_index].interp_count; - assert((interp_count == 0) == - (_PyRuntime.types.managed_static.types[full_index].type == NULL)); - *p_interp_count = interp_count; + assert(index < _Py_MAX_MANAGED_STATIC_TYPES); + assert(index >= _PyRuntime.types.managed_static.num_builtins); + assert(!_PyRuntime.types.managed_static.types[index].isbuiltin); - PyTypeObject *type = interp->types.for_extensions.initialized[index].type; + PyTypeObject *type = _PyRuntime.types.managed_static.types[index].type; + int64_t count = _PyRuntime.types.managed_static.types[index].interp_count; if (type == NULL) { + /* It was already cleared. */ + assert(count == 0); + *p_interp_count = count; return NULL; } - assert(!interp->types.for_extensions.initialized[index].isbuiltin); - assert(type == _PyRuntime.types.managed_static.types[full_index].type); + assert(count > 0); + assert(interp->types.managed_static.types[index].type == type); assert(managed_static_type_index_is_set(type)); + + *p_interp_count = count; return type; } static managed_static_type_state * managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self) { - // It's probably a builtin type. size_t index = managed_static_type_index_get(self); - managed_static_type_state *state = - &(interp->types.builtins.initialized[index]); - if (state->type == self) { - return state; - } - if (index > _Py_MAX_MANAGED_STATIC_EXT_TYPES) { - return state; - } - return &(interp->types.for_extensions.initialized[index]); + return &(interp->types.managed_static.types[index]); } /* For static types we store some state in an array on each interpreter. */ @@ -213,69 +227,105 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int initial) { assert(interp->runtime == &_PyRuntime); + assert(!isbuiltin || + ((initial == 0) == (_Py_IsMainInterpreter(interp) == 0))); + /* Initialize the index. */ size_t index; if (initial) { - assert(!managed_static_type_index_is_set(self)); - if (isbuiltin) { - index = interp->types.builtins.num_initialized; - assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + if (managed_static_type_index_is_set(self)) { + /* The runtime was re-initialized. */ + index = managed_static_type_index_reinit(self); } else { - PyMutex_Lock(&interp->types.mutex); - index = interp->types.for_extensions.next_index; - interp->types.for_extensions.next_index++; - PyMutex_Unlock(&interp->types.mutex); - assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + index = managed_static_type_index_init(self); } - managed_static_type_index_set(self, index); } else { index = managed_static_type_index_get(self); + } + managed_static_type_state *state = + &(interp->types.managed_static.types[index]); + + /* Check the existing global state. */ + if (initial) { if (isbuiltin) { - assert(index == interp->types.builtins.num_initialized); - assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES); + assert(_PyRuntime.types.managed_static.num_builtins == + _PyRuntime.types.managed_static.num_types); + assert(index == _PyRuntime.types.managed_static.num_builtins); + assert(index == _PyRuntime.types.managed_static.num_types); + /* All builtin static types are initialized during runtime init, + before any extension module static type is initialized, + and always in the same order. */ + assert(index == 0 || + _PyRuntime.types.managed_static.types[index-1].isbuiltin); } else { - assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); + assert(_PyRuntime.types.managed_static.num_builtins <= + _PyRuntime.types.managed_static.num_types); + assert(index >= _PyRuntime.types.managed_static.num_builtins); + /* There may be gaps for ext. module static types. */ + assert(index <= _PyRuntime.types.managed_static.num_types); } + assert(_PyRuntime.types.managed_static.types[index].type == NULL); + assert(_PyRuntime.types.managed_static.types[index].interp_count == 0); + } + else { + assert(_PyRuntime.types.managed_static.types[index].type == self); + assert(_PyRuntime.types.managed_static.types[index].isbuiltin == isbuiltin); + assert(_PyRuntime.types.managed_static.types[index].interp_count > 0); } - size_t full_index = isbuiltin - ? index - : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - - assert((initial == 1) == - (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); - (void)_Py_atomic_add_int64( - &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); + /* Check the existing per-interpreter state. */ if (initial) { - assert(_PyRuntime.types.managed_static.types[full_index].type == NULL); - _PyRuntime.types.managed_static.types[full_index].type = self; + assert(_PyRuntime.types.managed_static.num_builtins == + interp->types.managed_static.num_builtins); + assert(_PyRuntime.types.managed_static.num_types == + interp->types.managed_static.num_types); + if (isbuiltin) { + assert(index == interp->types.managed_static.num_builtins); + assert(index == interp->types.managed_static.num_types); + } + else { + assert(index >= interp->types.managed_static.num_builtins); + /* There may be gaps for ext. module static types. */ + assert(index <= interp->types.managed_static.num_types); + } } else { - assert(_PyRuntime.types.managed_static.types[full_index].type == self); } + assert(state->type == NULL); + assert(!state->readying && !state->ready); + assert(state->tp_dict == NULL); + assert(state->tp_subclasses == NULL); + assert(state->tp_weaklist == NULL); - managed_static_type_state *state = isbuiltin - ? &(interp->types.builtins.initialized[index]) - : &(interp->types.for_extensions.initialized[index]); + /* Update the global state. */ + if (initial) { + if (isbuiltin) { + _PyRuntime.types.managed_static.num_types += 1; + _PyRuntime.types.managed_static.num_builtins += 1; + } + else { + PyMutex_Lock(&_PyRuntime.types.managed_static.mutex); + _PyRuntime.types.managed_static.num_types += 1; + PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex); + } + } + _PyRuntime.types.managed_static.types[index].type = self; + _PyRuntime.types.managed_static.types[index].isbuiltin = isbuiltin; + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[index].interp_count, 1); - /* It should only be called once for each builtin type per interpreter. */ - assert(state->type == NULL); + /* Update the per-interpreter state. */ + interp->types.managed_static.num_types += 1; + if (isbuiltin) { + interp->types.managed_static.num_builtins += 1; + } state->type = self; - state->isbuiltin = isbuiltin; - /* state->tp_subclasses is left NULL until init_subclasses() sets it. */ /* state->tp_weaklist is left NULL until insert_head() or insert_after() (in weakrefobject.c) sets it. */ - - if (isbuiltin) { - interp->types.builtins.num_initialized++; - } - else { - interp->types.for_extensions.num_initialized++; - } } /* Reset the type's per-interpreter state. @@ -284,44 +334,74 @@ static void managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, int isbuiltin, int final) { - size_t index = managed_static_type_index_get(self); - size_t full_index = isbuiltin - ? index - : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; + assert(interp->runtime == &_PyRuntime); - managed_static_type_state *state = isbuiltin - ? &(interp->types.builtins.initialized[index]) - : &(interp->types.for_extensions.initialized[index]); + size_t index = managed_static_type_index_get(self); + if (isbuiltin) { + assert(!final + || _PyRuntime.types.managed_static.num_types == + _PyRuntime.types.managed_static.num_builtins); + assert(_PyRuntime.types.managed_static.types[index].isbuiltin); + } + else { + assert(_PyRuntime.types.managed_static.num_types > + _PyRuntime.types.managed_static.num_builtins); + assert(!_PyRuntime.types.managed_static.types[index].isbuiltin); + } + managed_static_type_state *state = + &(interp->types.managed_static.types[index]); assert(state != NULL); - assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); - assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); + assert(_PyRuntime.types.managed_static.types[index].type == state->type); + assert(_PyRuntime.types.managed_static.types[index].interp_count > 0); + (void)_Py_atomic_add_int64( + &_PyRuntime.types.managed_static.types[index].interp_count, -1); - assert(state->type != NULL); + /* Update the various counts. */ + interp->types.managed_static.num_types -= 1; + if (isbuiltin) { + interp->types.managed_static.num_builtins -= 1; + } + if (final) { + if (isbuiltin) { + _PyRuntime.types.managed_static.num_types -= 1; + _PyRuntime.types.managed_static.num_builtins -= 1; + } + else { + PyMutex_Lock(&_PyRuntime.types.managed_static.mutex); + _PyRuntime.types.managed_static.num_types -= 1; + PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex); + } + } + + /* Clear the state. */ + assert(_PyRuntime.types.managed_static.types[index].type == self); + assert(state->type == self); state->type = NULL; assert(state->tp_weaklist == NULL); // It was already cleared out. - - (void)_Py_atomic_add_int64( - &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); if (final) { - assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); - _PyRuntime.types.managed_static.types[full_index].type = NULL; - + assert(!_PyRuntime.types.managed_static.types[index].interp_count); + _PyRuntime.types.managed_static.types[index].type = NULL; + _PyRuntime.types.managed_static.types[index].isbuiltin = 0; managed_static_type_index_clear(self); } +} - if (isbuiltin) { - assert(interp->types.builtins.num_initialized > 0); - interp->types.builtins.num_initialized--; +static void +managed_static_types_fini(PyInterpreterState *interp) +{ + assert(interp->types.managed_static.num_types == 0); + // All the managed static types should have been finalized already. + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_TYPES; i++) { + assert(interp->types.managed_static.types[i].type == NULL); } - else { - PyMutex_Lock(&interp->types.mutex); - assert(interp->types.for_extensions.num_initialized > 0); - interp->types.for_extensions.num_initialized--; - if (interp->types.for_extensions.num_initialized == 0) { - interp->types.for_extensions.next_index = 0; + if (_Py_IsMainInterpreter(interp)) { + assert(_PyRuntime.types.managed_static.num_types == 0); + // All the managed static types should have been finalized already. + for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_TYPES; i++) { + assert(_PyRuntime.types.managed_static.types[i].type == NULL); + assert(_PyRuntime.types.managed_static.types[i].interp_count == 0); } - PyMutex_Unlock(&interp->types.mutex); } } @@ -882,15 +962,7 @@ _PyTypes_Fini(PyInterpreterState *interp) struct type_cache *cache = &interp->types.type_cache; type_cache_clear(cache, NULL); - // All the managed static types should have been finalized already. - assert(interp->types.for_extensions.num_initialized == 0); - for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { - assert(interp->types.for_extensions.initialized[i].type == NULL); - } - assert(interp->types.builtins.num_initialized == 0); - for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { - assert(interp->types.builtins.initialized[i].type == NULL); - } + managed_static_types_fini(interp); } @@ -5882,17 +5954,30 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, void _PyTypes_FiniExtTypes(PyInterpreterState *interp) { - for (size_t i = _Py_MAX_MANAGED_STATIC_EXT_TYPES; i > 0; i--) { - if (interp->types.for_extensions.num_initialized == 0) { + /* Static extension types must be finalized before the builtins. */ + assert(interp->types.managed_static.num_builtins > 0); +#ifndef NDEBUG + int started = 0; +#endif + for (size_t i = _Py_MAX_MANAGED_STATIC_TYPES; i > 0; i--) + { + /* There may be gaps. */ + if (interp->types.managed_static.num_types == + interp->types.managed_static.num_builtins) + { break; } int64_t count = 0; PyTypeObject *type = static_ext_type_lookup(interp, i-1, &count); - if (type == NULL) { - continue; + assert(!started || type != NULL); + /* Currently there is no other way to finalize one of these types. */ + if (type != NULL) { +#ifndef NDEBUG + started = 1; +#endif + int final = (count == 1); + fini_static_type(interp, type, 0, final); } - int final = (count == 1); - fini_static_type(interp, type, 0, final); } }