Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update semantic analyzer for TypeVar defaults (PEP 696) #14873

Merged
merged 4 commits into from
Jun 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion mypy/message_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ def with_additional_msg(self, info: str) -> ErrorMessage:
INVALID_TYPEVAR_ARG_BOUND: Final = 'Type argument {} of "{}" must be a subtype of {}'
INVALID_TYPEVAR_ARG_VALUE: Final = 'Invalid type argument value for "{}"'
TYPEVAR_VARIANCE_DEF: Final = 'TypeVar "{}" may only be a literal bool'
TYPEVAR_BOUND_MUST_BE_TYPE: Final = 'TypeVar "bound" must be a type'
TYPEVAR_ARG_MUST_BE_TYPE: Final = '{} "{}" must be a type'
TYPEVAR_UNEXPECTED_ARGUMENT: Final = 'Unexpected argument to "TypeVar()"'
UNBOUND_TYPEVAR: Final = (
"A function returning TypeVar should receive at least "
Expand Down
152 changes: 124 additions & 28 deletions mypy/semanal.py
Original file line number Diff line number Diff line change
Expand Up @@ -4135,28 +4135,15 @@ def process_typevar_parameters(
if has_values:
self.fail("TypeVar cannot have both values and an upper bound", context)
return None
try:
# We want to use our custom error message below, so we suppress
# the default error message for invalid types here.
analyzed = self.expr_to_analyzed_type(
param_value, allow_placeholder=True, report_invalid_types=False
)
if analyzed is None:
# Type variables are special: we need to place them in the symbol table
# soon, even if upper bound is not ready yet. Otherwise avoiding
# a "deadlock" in this common pattern would be tricky:
# T = TypeVar('T', bound=Custom[Any])
# class Custom(Generic[T]):
# ...
analyzed = PlaceholderType(None, [], context.line)
upper_bound = get_proper_type(analyzed)
if isinstance(upper_bound, AnyType) and upper_bound.is_from_error:
self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value)
# Note: we do not return 'None' here -- we want to continue
# using the AnyType as the upper bound.
except TypeTranslationError:
self.fail(message_registry.TYPEVAR_BOUND_MUST_BE_TYPE, param_value)
tv_arg = self.get_typevarlike_argument("TypeVar", param_name, param_value, context)
if tv_arg is None:
return None
upper_bound = tv_arg
elif param_name == "default":
tv_arg = self.get_typevarlike_argument(
"TypeVar", param_name, param_value, context, allow_unbound_tvars=True
)
default = tv_arg or AnyType(TypeOfAny.from_error)
elif param_name == "values":
# Probably using obsolete syntax with values=(...). Explain the current syntax.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(Not related to your PR) I wonder when this did work, may have been before typing was in CPython

self.fail('TypeVar "values" argument not supported', context)
Expand Down Expand Up @@ -4184,6 +4171,52 @@ def process_typevar_parameters(
variance = INVARIANT
return variance, upper_bound, default

def get_typevarlike_argument(
self,
typevarlike_name: str,
param_name: str,
param_value: Expression,
context: Context,
*,
allow_unbound_tvars: bool = False,
allow_param_spec_literals: bool = False,
report_invalid_typevar_arg: bool = True,
) -> ProperType | None:
try:
# We want to use our custom error message below, so we suppress
# the default error message for invalid types here.
analyzed = self.expr_to_analyzed_type(
param_value,
allow_placeholder=True,
report_invalid_types=False,
allow_unbound_tvars=allow_unbound_tvars,
allow_param_spec_literals=allow_param_spec_literals,
)
if analyzed is None:
# Type variables are special: we need to place them in the symbol table
# soon, even if upper bound is not ready yet. Otherwise avoiding
# a "deadlock" in this common pattern would be tricky:
# T = TypeVar('T', bound=Custom[Any])
# class Custom(Generic[T]):
# ...
analyzed = PlaceholderType(None, [], context.line)
typ = get_proper_type(analyzed)
if report_invalid_typevar_arg and isinstance(typ, AnyType) and typ.is_from_error:
self.fail(
message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name),
param_value,
)
# Note: we do not return 'None' here -- we want to continue
# using the AnyType.
return typ
except TypeTranslationError:
if report_invalid_typevar_arg:
self.fail(
message_registry.TYPEVAR_ARG_MUST_BE_TYPE.format(typevarlike_name, param_name),
param_value,
)
return None

def extract_typevarlike_name(self, s: AssignmentStmt, call: CallExpr) -> str | None:
if not call:
return None
Expand Down Expand Up @@ -4216,13 +4249,50 @@ def process_paramspec_declaration(self, s: AssignmentStmt) -> bool:
if name is None:
return False

# ParamSpec is different from a regular TypeVar:
# arguments are not semantically valid. But, allowed in runtime.
# So, we need to warn users about possible invalid usage.
if len(call.args) > 1:
self.fail("Only the first argument to ParamSpec has defined semantics", s)
n_values = call.arg_kinds[1:].count(ARG_POS)
if n_values != 0:
self.fail('Too many positional arguments for "ParamSpec"', s)

default: Type = AnyType(TypeOfAny.from_omitted_generics)
for param_value, param_name in zip(
call.args[1 + n_values :], call.arg_names[1 + n_values :]
):
if param_name == "default":
tv_arg = self.get_typevarlike_argument(
"ParamSpec",
param_name,
param_value,
s,
allow_unbound_tvars=True,
allow_param_spec_literals=True,
report_invalid_typevar_arg=False,
)
default = tv_arg or AnyType(TypeOfAny.from_error)
if isinstance(tv_arg, Parameters):
for i, arg_type in enumerate(tv_arg.arg_types):
typ = get_proper_type(arg_type)
if isinstance(typ, AnyType) and typ.is_from_error:
self.fail(
f"Argument {i} of ParamSpec default must be a type", param_value
)
elif (
isinstance(default, AnyType)
and default.is_from_error
or not isinstance(default, (AnyType, UnboundType))
):
self.fail(
"The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec",
param_value,
)
default = AnyType(TypeOfAny.from_error)
else:
# ParamSpec is different from a regular TypeVar:
# arguments are not semantically valid. But, allowed in runtime.
# So, we need to warn users about possible invalid usage.
self.fail(
"The variance and bound arguments to ParamSpec do not have defined semantics yet",
s,
)

# PEP 612 reserves the right to define bound, covariant and contravariant arguments to
# ParamSpec in a later PEP. If and when that happens, we should do something
Expand Down Expand Up @@ -4256,10 +4326,32 @@ def process_typevartuple_declaration(self, s: AssignmentStmt) -> bool:
if not call:
return False

if len(call.args) > 1:
self.fail("Only the first argument to TypeVarTuple has defined semantics", s)
n_values = call.arg_kinds[1:].count(ARG_POS)
if n_values != 0:
self.fail('Too many positional arguments for "TypeVarTuple"', s)

default: Type = AnyType(TypeOfAny.from_omitted_generics)
for param_value, param_name in zip(
call.args[1 + n_values :], call.arg_names[1 + n_values :]
):
if param_name == "default":
tv_arg = self.get_typevarlike_argument(
"TypeVarTuple",
param_name,
param_value,
s,
allow_unbound_tvars=True,
report_invalid_typevar_arg=False,
)
default = tv_arg or AnyType(TypeOfAny.from_error)
if not isinstance(default, UnpackType):
self.fail(
"The default argument to TypeVarTuple must be an Unpacked tuple",
param_value,
)
default = AnyType(TypeOfAny.from_error)
else:
self.fail(f'Unexpected keyword argument "{param_name}" for "TypeVarTuple"', s)

if not self.incomplete_feature_enabled(TYPE_VAR_TUPLE, s):
return False
Expand Down Expand Up @@ -6359,6 +6451,8 @@ def expr_to_analyzed_type(
report_invalid_types: bool = True,
allow_placeholder: bool = False,
allow_type_any: bool = False,
allow_unbound_tvars: bool = False,
allow_param_spec_literals: bool = False,
) -> Type | None:
if isinstance(expr, CallExpr):
# This is a legacy syntax intended mostly for Python 2, we keep it for
Expand Down Expand Up @@ -6387,6 +6481,8 @@ def expr_to_analyzed_type(
report_invalid_types=report_invalid_types,
allow_placeholder=allow_placeholder,
allow_type_any=allow_type_any,
allow_unbound_tvars=allow_unbound_tvars,
allow_param_spec_literals=allow_param_spec_literals,
)

def analyze_type_expr(self, expr: Expression) -> None:
Expand Down
22 changes: 18 additions & 4 deletions mypy/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -3049,6 +3049,8 @@ def visit_type_var(self, t: TypeVarType) -> str:
s = f"{t.name}`{t.id}"
if self.id_mapper and t.upper_bound:
s += f"(upper_bound={t.upper_bound.accept(self)})"
if t.has_default():
s += f" = {t.default.accept(self)}"
return s

def visit_param_spec(self, t: ParamSpecType) -> str:
Expand All @@ -3064,6 +3066,8 @@ def visit_param_spec(self, t: ParamSpecType) -> str:
s += f"{t.name_with_suffix()}`{t.id}"
if t.prefix.arg_types:
s += "]"
if t.has_default():
s += f" = {t.default.accept(self)}"
return s

def visit_parameters(self, t: Parameters) -> str:
Expand Down Expand Up @@ -3102,6 +3106,8 @@ def visit_type_var_tuple(self, t: TypeVarTupleType) -> str:
else:
# Named type variable type.
s = f"{t.name}`{t.id}"
if t.has_default():
s += f" = {t.default.accept(self)}"
return s

def visit_callable_type(self, t: CallableType) -> str:
Expand Down Expand Up @@ -3138,6 +3144,8 @@ def visit_callable_type(self, t: CallableType) -> str:
if s:
s += ", "
s += f"*{n}.args, **{n}.kwargs"
if param_spec.has_default():
s += f" = {param_spec.default.accept(self)}"

s = f"({s})"

Expand All @@ -3156,12 +3164,18 @@ def visit_callable_type(self, t: CallableType) -> str:
vals = f"({', '.join(val.accept(self) for val in var.values)})"
vs.append(f"{var.name} in {vals}")
elif not is_named_instance(var.upper_bound, "builtins.object"):
vs.append(f"{var.name} <: {var.upper_bound.accept(self)}")
vs.append(
f"{var.name} <: {var.upper_bound.accept(self)}{f' = {var.default.accept(self)}' if var.has_default() else ''}"
)
else:
vs.append(var.name)
vs.append(
f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}"
)
else:
# For other TypeVarLikeTypes, just use the name
vs.append(var.name)
# For other TypeVarLikeTypes, use the name and default
vs.append(
f"{var.name}{f' = {var.default.accept(self)}' if var.has_default() else ''}"
)
s = f"[{', '.join(vs)}] {s}"

return f"def {s}"
Expand Down
10 changes: 5 additions & 5 deletions test-data/unit/check-parameter-specification.test
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ P = ParamSpec('P')
[case testInvalidParamSpecDefinitions]
from typing import ParamSpec

P1 = ParamSpec("P1", covariant=True) # E: Only the first argument to ParamSpec has defined semantics
P2 = ParamSpec("P2", contravariant=True) # E: Only the first argument to ParamSpec has defined semantics
P3 = ParamSpec("P3", bound=int) # E: Only the first argument to ParamSpec has defined semantics
P4 = ParamSpec("P4", int, str) # E: Only the first argument to ParamSpec has defined semantics
P5 = ParamSpec("P5", covariant=True, bound=int) # E: Only the first argument to ParamSpec has defined semantics
P1 = ParamSpec("P1", covariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
P2 = ParamSpec("P2", contravariant=True) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
P3 = ParamSpec("P3", bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
P4 = ParamSpec("P4", int, str) # E: Too many positional arguments for "ParamSpec"
P5 = ParamSpec("P5", covariant=True, bound=int) # E: The variance and bound arguments to ParamSpec do not have defined semantics yet
[builtins fixtures/paramspec.pyi]

[case testParamSpecLocations]
Expand Down
74 changes: 74 additions & 0 deletions test-data/unit/check-typevar-defaults.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
[case testTypeVarDefaultsBasic]
import builtins
from typing import Generic, TypeVar, ParamSpec, Callable, Tuple, List
from typing_extensions import TypeVarTuple, Unpack

T1 = TypeVar("T1", default=int)
P1 = ParamSpec("P1", default=[int, str])
Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int, str]])

def f1(a: T1) -> List[T1]: ...
reveal_type(f1) # N: Revealed type is "def [T1 = builtins.int] (a: T1`-1 = builtins.int) -> builtins.list[T1`-1 = builtins.int]"

def f2(a: Callable[P1, None] ) -> Callable[P1, None]: ...
reveal_type(f2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] (a: def (*P1.args, **P1.kwargs)) -> def (*P1.args, **P1.kwargs)"

def f3(a: Tuple[Unpack[Ts1]]) -> Tuple[Unpack[Ts1]]: ...
reveal_type(f3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] (a: Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]) -> Tuple[Unpack[Ts1`-1 = Unpack[Tuple[builtins.int, builtins.str]]]]"


class ClassA1(Generic[T1]): ...
class ClassA2(Generic[P1]): ...
class ClassA3(Generic[Unpack[Ts1]]): ...

reveal_type(ClassA1) # N: Revealed type is "def [T1 = builtins.int] () -> __main__.ClassA1[T1`1 = builtins.int]"
reveal_type(ClassA2) # N: Revealed type is "def [P1 = [builtins.int, builtins.str]] () -> __main__.ClassA2[P1`1 = [builtins.int, builtins.str]]"
reveal_type(ClassA3) # N: Revealed type is "def [Ts1 = Unpack[Tuple[builtins.int, builtins.str]]] () -> __main__.ClassA3[Unpack[Ts1`1 = Unpack[Tuple[builtins.int, builtins.str]]]]"
[builtins fixtures/tuple.pyi]

[case testTypeVarDefaultsValid]
from typing import TypeVar, ParamSpec, Any, List, Tuple
from typing_extensions import TypeVarTuple, Unpack

S0 = TypeVar("S0")
S1 = TypeVar("S1", bound=int)

P0 = ParamSpec("P0")
Ts0 = TypeVarTuple("Ts0")

T1 = TypeVar("T1", default=int)
T2 = TypeVar("T2", bound=float, default=int)
T3 = TypeVar("T3", bound=List[Any], default=List[int])
T4 = TypeVar("T4", int, str, default=int)
T5 = TypeVar("T5", default=S0)
T6 = TypeVar("T6", bound=float, default=S1)
# T7 = TypeVar("T7", bound=List[Any], default=List[S0]) # TODO

P1 = ParamSpec("P1", default=[])
P2 = ParamSpec("P2", default=...)
P3 = ParamSpec("P3", default=[int, str])
P4 = ParamSpec("P4", default=P0)

Ts1 = TypeVarTuple("Ts1", default=Unpack[Tuple[int]])
Ts2 = TypeVarTuple("Ts2", default=Unpack[Tuple[int, ...]])
# Ts3 = TypeVarTuple("Ts3", default=Unpack[Ts0]) # TODO
[builtins fixtures/tuple.pyi]

[case testTypeVarDefaultsInvalid]
from typing import TypeVar, ParamSpec, Tuple
from typing_extensions import TypeVarTuple, Unpack

T1 = TypeVar("T1", default=2) # E: TypeVar "default" must be a type
T2 = TypeVar("T2", default=[int, str]) # E: Bracketed expression "[...]" is not valid as a type \
# N: Did you mean "List[...]"? \
# E: TypeVar "default" must be a type

P1 = ParamSpec("P1", default=int) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
P2 = ParamSpec("P2", default=2) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
P3 = ParamSpec("P3", default=(2, int)) # E: The default argument to ParamSpec must be a list expression, ellipsis, or a ParamSpec
P4 = ParamSpec("P4", default=[2, int]) # E: Argument 0 of ParamSpec default must be a type

Ts1 = TypeVarTuple("Ts1", default=2) # E: The default argument to TypeVarTuple must be an Unpacked tuple
Ts2 = TypeVarTuple("Ts2", default=int) # E: The default argument to TypeVarTuple must be an Unpacked tuple
Ts3 = TypeVarTuple("Ts3", default=Tuple[int]) # E: The default argument to TypeVarTuple must be an Unpacked tuple
[builtins fixtures/tuple.pyi]
3 changes: 2 additions & 1 deletion test-data/unit/semanal-errors.test
Original file line number Diff line number Diff line change
Expand Up @@ -1465,8 +1465,9 @@ TVariadic2 = TypeVarTuple('TVariadic2')
TP = TypeVarTuple('?') # E: String argument 1 "?" to TypeVarTuple(...) does not match variable name "TP"
TP2: int = TypeVarTuple('TP2') # E: Cannot declare the type of a TypeVar or similar construct
TP3 = TypeVarTuple() # E: Too few arguments for TypeVarTuple()
TP4 = TypeVarTuple('TP4', 'TP4') # E: Only the first argument to TypeVarTuple has defined semantics
TP4 = TypeVarTuple('TP4', 'TP4') # E: Too many positional arguments for "TypeVarTuple"
TP5 = TypeVarTuple(t='TP5') # E: TypeVarTuple() expects a string literal as first argument
TP6 = TypeVarTuple('TP6', bound=int) # E: Unexpected keyword argument "bound" for "TypeVarTuple"

x: TVariadic # E: TypeVarTuple "TVariadic" is unbound
y: Unpack[TVariadic] # E: TypeVarTuple "TVariadic" is unbound
Expand Down