diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 755d69a873cdcd..35bffa7aff9493 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -199,6 +199,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__subclasshook__) STRUCT_FOR_ID(__truediv__) STRUCT_FOR_ID(__trunc__) + STRUCT_FOR_ID(__typing_subst__) STRUCT_FOR_ID(__warningregistry__) STRUCT_FOR_ID(__weakref__) STRUCT_FOR_ID(__xor__) diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 5ba18267aeb349..20d543a8cbc565 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -822,6 +822,7 @@ extern "C" { INIT_ID(__subclasshook__), \ INIT_ID(__truediv__), \ INIT_ID(__trunc__), \ + INIT_ID(__typing_subst__), \ INIT_ID(__warningregistry__), \ INIT_ID(__weakref__), \ INIT_ID(__xor__), \ diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 40417dc1d3133d..86eb042e3a75ab 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -430,25 +430,13 @@ def __new__(cls, origin, args): raise TypeError( "Callable must be used as Callable[[arg, ...], result].") t_args, t_result = args - if isinstance(t_args, list): + if isinstance(t_args, (tuple, list)): args = (*t_args, t_result) elif not _is_param_expr(t_args): raise TypeError(f"Expected a list of types, an ellipsis, " f"ParamSpec, or Concatenate. Got {t_args}") return super().__new__(cls, origin, args) - @property - def __parameters__(self): - params = [] - for arg in self.__args__: - # Looks like a genericalias - if hasattr(arg, "__parameters__") and isinstance(arg.__parameters__, tuple): - params.extend(arg.__parameters__) - else: - if _is_typevarlike(arg): - params.append(arg) - return tuple(dict.fromkeys(params)) - def __repr__(self): if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]): return super().__repr__() @@ -468,57 +456,24 @@ def __getitem__(self, item): # code is copied from typing's _GenericAlias and the builtin # types.GenericAlias. - # A special case in PEP 612 where if X = Callable[P, int], - # then X[int, str] == X[[int, str]]. - param_len = len(self.__parameters__) - if param_len == 0: - raise TypeError(f'{self} is not a generic class') if not isinstance(item, tuple): item = (item,) - if (param_len == 1 and _is_param_expr(self.__parameters__[0]) + # A special case in PEP 612 where if X = Callable[P, int], + # then X[int, str] == X[[int, str]]. + if (len(self.__parameters__) == 1 + and _is_param_expr(self.__parameters__[0]) and item and not _is_param_expr(item[0])): - item = (list(item),) - item_len = len(item) - if item_len != param_len: - raise TypeError(f'Too {"many" if item_len > param_len else "few"}' - f' arguments for {self};' - f' actual {item_len}, expected {param_len}') - subst = dict(zip(self.__parameters__, item)) - new_args = [] - for arg in self.__args__: - if _is_typevarlike(arg): - if _is_param_expr(arg): - arg = subst[arg] - if not _is_param_expr(arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {arg}") - else: - arg = subst[arg] - # Looks like a GenericAlias - elif hasattr(arg, '__parameters__') and isinstance(arg.__parameters__, tuple): - subparams = arg.__parameters__ - if subparams: - subargs = tuple(subst[x] for x in subparams) - arg = arg[subargs] - if isinstance(arg, tuple): - new_args.extend(arg) - else: - new_args.append(arg) + item = (item,) + + new_args = super().__getitem__(item).__args__ # args[0] occurs due to things like Z[[int, str, bool]] from PEP 612 - if not isinstance(new_args[0], list): + if not isinstance(new_args[0], (tuple, list)): t_result = new_args[-1] t_args = new_args[:-1] new_args = (t_args, t_result) return _CallableGenericAlias(Callable, tuple(new_args)) - -def _is_typevarlike(arg): - obj = type(arg) - # looks like a TypeVar/ParamSpec - return (obj.__module__ == 'typing' - and obj.__name__ in {'ParamSpec', 'TypeVar'}) - def _is_param_expr(obj): """Checks if obj matches either a list of types, ``...``, ``ParamSpec`` or ``_ConcatenateGenericAlias`` from typing.py diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index fc596e4d90b210..91b2e77e97b5a3 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -360,10 +360,31 @@ def test_no_bivariant(self): with self.assertRaises(ValueError): TypeVar('T', covariant=True, contravariant=True) + def test_var_substitution(self): + T = TypeVar('T') + subst = T.__typing_subst__ + self.assertIs(subst(int), int) + self.assertEqual(subst(list[int]), list[int]) + self.assertEqual(subst(List[int]), List[int]) + self.assertEqual(subst(List), List) + self.assertIs(subst(Any), Any) + self.assertIs(subst(None), type(None)) + self.assertIs(subst(T), T) + self.assertEqual(subst(int|str), int|str) + self.assertEqual(subst(Union[int, str]), Union[int, str]) + def test_bad_var_substitution(self): T = TypeVar('T') - for arg in (), (int, str): + P = ParamSpec("P") + bad_args = ( + 42, ..., [int], (), (int, str), Union, + Generic, Generic[T], Protocol, Protocol[T], + Final, Final[int], ClassVar, ClassVar[int], + ) + for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + T.__typing_subst__(arg) with self.assertRaises(TypeError): List[T][arg] with self.assertRaises(TypeError): @@ -1110,8 +1131,7 @@ def test_var_substitution(self): C2 = Callable[[KT, T], VT] C3 = Callable[..., T] self.assertEqual(C1[str], Callable[[int, str], str]) - if Callable is typing.Callable: - self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) + self.assertEqual(C1[None], Callable[[int, type(None)], type(None)]) self.assertEqual(C2[int, float, str], Callable[[int, float], str]) self.assertEqual(C3[int], Callable[..., int]) self.assertEqual(C3[NoReturn], Callable[..., NoReturn]) @@ -2696,7 +2716,10 @@ def test_all_repr_eq_any(self): for obj in objs: self.assertNotEqual(repr(obj), '') self.assertEqual(obj, obj) - if getattr(obj, '__parameters__', None) and len(obj.__parameters__) == 1: + if (getattr(obj, '__parameters__', None) + and not isinstance(obj, typing.TypeVar) + and isinstance(obj.__parameters__, tuple) + and len(obj.__parameters__) == 1): self.assertEqual(obj[Any].__args__, (Any,)) if isinstance(obj, type): for base in obj.__mro__: @@ -5748,33 +5771,30 @@ class X(Generic[P, P2]): self.assertEqual(G1.__args__, ((int, str), (bytes,))) self.assertEqual(G2.__args__, ((int,), (str, bytes))) + def test_var_substitution(self): + T = TypeVar("T") + P = ParamSpec("P") + subst = P.__typing_subst__ + self.assertEqual(subst((int, str)), (int, str)) + self.assertEqual(subst([int, str]), (int, str)) + self.assertEqual(subst([None]), (type(None),)) + self.assertIs(subst(...), ...) + self.assertIs(subst(P), P) + self.assertEqual(subst(Concatenate[int, P]), Concatenate[int, P]) + def test_bad_var_substitution(self): T = TypeVar('T') P = ParamSpec('P') bad_args = (42, int, None, T, int|str, Union[int, str]) for arg in bad_args: with self.subTest(arg=arg): + with self.assertRaises(TypeError): + P.__typing_subst__(arg) with self.assertRaises(TypeError): typing.Callable[P, T][arg, str] with self.assertRaises(TypeError): collections.abc.Callable[P, T][arg, str] - def test_no_paramspec_in__parameters__(self): - # ParamSpec should not be found in __parameters__ - # of generics. Usages outside Callable, Concatenate - # and Generic are invalid. - T = TypeVar("T") - P = ParamSpec("P") - self.assertNotIn(P, List[P].__parameters__) - self.assertIn(T, Tuple[T, P].__parameters__) - - # Test for consistency with builtin generics. - self.assertNotIn(P, list[P].__parameters__) - self.assertIn(T, tuple[T, P].__parameters__) - - self.assertNotIn(P, (list[P] | int).__parameters__) - self.assertIn(T, (tuple[T, P] | int).__parameters__) - def test_paramspec_in_nested_generics(self): # Although ParamSpec should not be found in __parameters__ of most # generics, they probably should be found when nested in diff --git a/Lib/typing.py b/Lib/typing.py index e3015563b3e8ce..062c01ef2a9b9e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -179,7 +179,9 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") - if arg in (Any, NoReturn, Never, Self, ClassVar, Final, TypeAlias): + if arg in (Any, NoReturn, Never, Self, TypeAlias): + return arg + if allow_special_forms and arg in (ClassVar, Final): return arg if isinstance(arg, _SpecialForm) or arg in (Generic, Protocol): raise TypeError(f"Plain {arg} is not valid as type argument") @@ -217,21 +219,22 @@ def _type_repr(obj): return repr(obj) -def _collect_type_vars(types_, typevar_types=None): - """Collect all type variable contained - in types in order of first appearance (lexicographic order). For example:: +def _collect_parameters(args): + """Collect all type variables and parameter specifications in args + in order of first appearance (lexicographic order). For example:: - _collect_type_vars((T, List[S, T])) == (T, S) + _collect_parameters((T, Callable[P, T])) == (T, P) """ - if typevar_types is None: - typevar_types = TypeVar - tvars = [] - for t in types_: - if isinstance(t, typevar_types) and t not in tvars: - tvars.append(t) - if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - return tuple(tvars) + parameters = [] + for t in args: + if hasattr(t, '__typing_subst__'): + if t not in parameters: + parameters.append(t) + else: + for x in getattr(t, '__parameters__', ()): + if x not in parameters: + parameters.append(x) + return tuple(parameters) def _check_generic(cls, parameters, elen): @@ -671,7 +674,6 @@ def Concatenate(self, parameters): msg = "Concatenate[arg, ...]: each arg must be a type." parameters = (*(_type_check(p, msg) for p in parameters[:-1]), parameters[-1]) return _ConcatenateGenericAlias(self, parameters, - _typevar_types=(TypeVar, ParamSpec), _paramspec_tvars=True) @@ -909,6 +911,11 @@ def __init__(self, name, *constraints, bound=None, if def_mod != 'typing': self.__module__ = def_mod + def __typing_subst__(self, arg): + msg = "Parameters to generic types must be types." + arg = _type_check(arg, msg, is_argument=True) + return arg + class TypeVarTuple(_Final, _Immutable, _root=True): """Type variable tuple. @@ -942,6 +949,9 @@ def __iter__(self): def __repr__(self): return self._name + def __typing_subst__(self, arg): + raise AssertionError + class ParamSpecArgs(_Final, _Immutable, _root=True): """The args for a ParamSpec object. @@ -1052,6 +1062,14 @@ def __init__(self, name, *, bound=None, covariant=False, contravariant=False): if def_mod != 'typing': self.__module__ = def_mod + def __typing_subst__(self, arg): + if isinstance(arg, (list, tuple)): + arg = tuple(_type_check(a, "Expected a type.") for a in arg) + elif not _is_param_expr(arg): + raise TypeError(f"Expected a list of types, an ellipsis, " + f"ParamSpec, or Concatenate. Got {arg}") + return arg + def _is_dunder(attr): return attr.startswith('__') and attr.endswith('__') @@ -1106,7 +1124,7 @@ def __getattr__(self, attr): def __setattr__(self, attr, val): if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', - '_typevar_types', '_paramspec_tvars'}: + '_paramspec_tvars'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1199,7 +1217,6 @@ class _GenericAlias(_BaseGenericAlias, _root=True): # TypeVar[bool] def __init__(self, origin, args, *, inst=True, name=None, - _typevar_types=(TypeVar, TypeVarTuple), _paramspec_tvars=False): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): @@ -1207,8 +1224,7 @@ def __init__(self, origin, args, *, inst=True, name=None, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) - self.__parameters__ = _collect_type_vars(args, typevar_types=_typevar_types) - self._typevar_types = _typevar_types + self.__parameters__ = _collect_parameters(args) self._paramspec_tvars = _paramspec_tvars if not name: self.__module__ = origin.__module__ @@ -1291,26 +1307,20 @@ def _determine_new_args(self, args): new_args = [] for old_arg in self.__args__: - if isinstance(old_arg, ParamSpec): - new_arg = new_arg_by_param[old_arg] - if not _is_param_expr(new_arg): - raise TypeError(f"Expected a list of types, an ellipsis, " - f"ParamSpec, or Concatenate. Got {new_arg}") - elif isinstance(old_arg, self._typevar_types): - new_arg = new_arg_by_param[old_arg] - elif (TypeVarTuple in self._typevar_types - and _is_unpacked_typevartuple(old_arg)): + if _is_unpacked_typevartuple(old_arg): original_typevartuple = old_arg.__parameters__[0] new_arg = new_arg_by_param[original_typevartuple] - elif isinstance(old_arg, (_GenericAlias, GenericAlias, types.UnionType)): - subparams = old_arg.__parameters__ - if not subparams: - new_arg = old_arg - else: - subargs = tuple(new_arg_by_param[x] for x in subparams) - new_arg = old_arg[subargs] else: - new_arg = old_arg + substfunc = getattr(old_arg, '__typing_subst__', None) + if substfunc: + new_arg = substfunc(new_arg_by_param[old_arg]) + else: + subparams = getattr(old_arg, '__parameters__', ()) + if not subparams: + new_arg = old_arg + else: + subargs = tuple(new_arg_by_param[x] for x in subparams) + new_arg = old_arg[subargs] if self.__origin__ == collections.abc.Callable and isinstance(new_arg, tuple): # Consider the following `Callable`. @@ -1342,7 +1352,6 @@ def _determine_new_args(self, args): def copy_with(self, args): return self.__class__(self.__origin__, args, name=self._name, inst=self._inst, - _typevar_types=self._typevar_types, _paramspec_tvars=self._paramspec_tvars) def __repr__(self): @@ -1454,7 +1463,6 @@ class _CallableType(_SpecialGenericAlias, _root=True): def copy_with(self, params): return _CallableGenericAlias(self.__origin__, params, name=self._name, inst=self._inst, - _typevar_types=(TypeVar, ParamSpec), _paramspec_tvars=True) def __getitem__(self, params): @@ -1675,11 +1683,8 @@ def __class_getitem__(cls, params): # don't check variadic generic arity at runtime (to reduce # complexity of typing.py). _check_generic(cls, params, len(cls.__parameters__)) - return _GenericAlias( - cls, params, - _typevar_types=(TypeVar, TypeVarTuple, ParamSpec), - _paramspec_tvars=True, - ) + return _GenericAlias(cls, params, + _paramspec_tvars=True) def __init_subclass__(cls, *args, **kwargs): super().__init_subclass__(*args, **kwargs) @@ -1691,9 +1696,7 @@ def __init_subclass__(cls, *args, **kwargs): if error: raise TypeError("Cannot inherit from plain Generic") if '__orig_bases__' in cls.__dict__: - tvars = _collect_type_vars( - cls.__orig_bases__, (TypeVar, TypeVarTuple, ParamSpec) - ) + tvars = _collect_parameters(cls.__orig_bases__) # Look for Generic[T1, ..., Tn]. # If found, tvars must be a subset of it. # If not found, tvars is it. diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index b41644910f5d25..45caf2e2ee7db0 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -152,25 +152,6 @@ ga_repr(PyObject *self) return NULL; } -// isinstance(obj, TypeVar) without importing typing.py. -// Returns -1 for errors. -static int -is_typevar(PyObject *obj) -{ - PyTypeObject *type = Py_TYPE(obj); - if (strcmp(type->tp_name, "TypeVar") != 0) { - return 0; - } - PyObject *module = PyObject_GetAttrString((PyObject *)type, "__module__"); - if (module == NULL) { - return -1; - } - int res = PyUnicode_Check(module) - && _PyUnicode_EqualToASCIIString(module, "typing"); - Py_DECREF(module); - return res; -} - // Index of item in self[:len], or -1 if not found (self is a tuple) static Py_ssize_t tuple_index(PyObject *self, Py_ssize_t len, PyObject *item) @@ -205,13 +186,14 @@ _Py_make_parameters(PyObject *args) Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *t = PyTuple_GET_ITEM(args, iarg); - int typevar = is_typevar(t); - if (typevar < 0) { + PyObject *subst; + if (_PyObject_LookupAttr(t, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(parameters); return NULL; } - if (typevar) { + if (subst) { iparam += tuple_add(parameters, iparam, t); + Py_DECREF(subst); } else { PyObject *subparams; @@ -295,7 +277,7 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje Py_ssize_t nparams = PyTuple_GET_SIZE(parameters); if (nparams == 0) { return PyErr_Format(PyExc_TypeError, - "There are no type variables left in %R", + "%R is not a generic class", self); } int is_tuple = PyTuple_Check(item); @@ -320,23 +302,23 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje } for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { PyObject *arg = PyTuple_GET_ITEM(args, iarg); - int typevar = is_typevar(arg); - if (typevar < 0) { + PyObject *subst; + if (_PyObject_LookupAttr(arg, &_Py_ID(__typing_subst__), &subst) < 0) { Py_DECREF(newargs); return NULL; } - if (typevar) { + if (subst) { Py_ssize_t iparam = tuple_index(parameters, nparams, arg); assert(iparam >= 0); - arg = argitems[iparam]; - Py_INCREF(arg); + arg = PyObject_CallOneArg(subst, argitems[iparam]); + Py_DECREF(subst); } else { arg = subs_tvars(arg, parameters, argitems); - if (arg == NULL) { - Py_DECREF(newargs); - return NULL; - } + } + if (arg == NULL) { + Py_DECREF(newargs); + return NULL; } PyTuple_SET_ITEM(newargs, iarg, arg); }