Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bpo-43224: Draft implementation of PEP 646 #30398

Closed
wants to merge 18 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Doc/whatsnew/3.11.rst
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ Summary -- Release highlights
PEP-654: Exception Groups and ``except*``.
(Contributed by Irit Katriel in :issue:`45292`.)

PEP 646: Variadic Generics.
(Contributed by Matthew Rahtz in :issue:`43224`.)

New Features
============

Expand Down
9 changes: 8 additions & 1 deletion Grammar/python.gram
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,8 @@ slash_with_default[SlashWithDefault*]:
star_etc[StarEtc*]:
| '*' a=param_no_default b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' a=param_no_default_star_annotation b=param_maybe_default* c=[kwds] {
_PyPegen_star_etc(p, a, b, c) }
| '*' ',' b=param_maybe_default+ c=[kwds] {
_PyPegen_star_etc(p, NULL, b, c) }
| a=kwds { _PyPegen_star_etc(p, NULL, NULL, a) }
Expand All @@ -331,14 +333,19 @@ kwds[arg_ty]: '**' a=param_no_default { a }
param_no_default[arg_ty]:
| a=param ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_no_default_star_annotation[arg_ty]:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not join these to param_no_default? That way we can remove the extra rule in star_etc.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, wouldn't that mess with other things that use param_no_default though? E.g. kwds uses param_no_default, so if we joined these to param_no_default, wouldn't we accidentally allow def foo(***args)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Friendly poke :)

| a=param_star_annotation ',' tc=TYPE_COMMENT? { _PyPegen_add_type_comment_to_arg(p, a, tc) }
| a=param_star_annotation tc=TYPE_COMMENT? &')' { _PyPegen_add_type_comment_to_arg(p, a, tc) }
param_with_default[NameDefaultPair*]:
| a=param c=default ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param_maybe_default[NameDefaultPair*]:
| a=param c=default? ',' tc=TYPE_COMMENT? { _PyPegen_name_default_pair(p, a, c, tc) }
| a=param c=default? tc=TYPE_COMMENT? &')' { _PyPegen_name_default_pair(p, a, c, tc) }
param[arg_ty]: a=NAME b=annotation? { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
param_star_annotation[arg_ty]: a=NAME b=star_annotation { _PyAST_arg(a->v.Name.id, b, NULL, EXTRA) }
annotation[expr_ty]: ':' a=expression { a }
star_annotation[expr_ty]: ':' a=star_expression { a }
default[expr_ty]: '=' a=expression { a }

# If statement
Expand Down Expand Up @@ -780,7 +787,7 @@ primary[expr_ty]:

slices[expr_ty]:
| a=slice !',' { a }
| a[asdl_expr_seq*]=','.slice+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
| a[asdl_expr_seq*]=','.(slice | starred_expression)+ [','] { _PyAST_Tuple(a, Load, EXTRA) }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer to move starred_expression as another option of slice as we already have named_expression there

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just tried this, but then I get SyntaxError: can't use starred expression here for e.g. a[*b].

I think what happens is, if we move starred_expression to slice (as | a=starred_expression { a }), then when we do a[*b], it triggers the first option of slices, where the starred_expression doesn't get processed by _PyAST_Tuple.

I can't see any way to make sure _PyAST_Tuple definitely gets called for starred_expression without doing something to slices, so I think the original attempt with (slice | starred_expression) is cleanest. Wdyt?


slice[expr_ty]:
| a=[expression] ':' b=[expression] c=[':' d=[expression] { d }] { _PyAST_Slice(a, b, c, EXTRA) }
Expand Down
25 changes: 24 additions & 1 deletion Lib/test/test_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from test import support

def to_tuple(t):
if t is None or isinstance(t, (str, int, complex)):
if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis:
mrahtz marked this conversation as resolved.
Show resolved Hide resolved
return t
elif isinstance(t, list):
return [to_tuple(e) for e in t]
Expand Down Expand Up @@ -46,10 +46,20 @@ def to_tuple(t):
"def f(a=0): pass",
# FunctionDef with varargs
"def f(*args): pass",
# FunctionDef with varargs as TypeVarTuple
"def f(*args: *Ts): pass",
# FunctionDef with varargs as unpacked Tuple
"def f(*args: *tuple[int, ...]): pass",
# FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple
"def f(*args: *tuple[int, *Ts]): pass",
# FunctionDef with kwargs
"def f(**kwargs): pass",
# FunctionDef with all kind of args and docstring
"def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'",
# FunctionDef with type annotation on return involving unpacking
"def f() -> tuple[*Ts]: pass",
"def f() -> tuple[int, *Ts]: pass",
"def f() -> tuple[int, *tuple[int, ...]]: pass",
# ClassDef
"class C:pass",
# ClassDef with docstring
Expand All @@ -65,6 +75,10 @@ def to_tuple(t):
"a,b = c",
"(a,b) = c",
"[a,b] = c",
# AnnAssign with unpacked types
"x: tuple[*Ts]",
"x: tuple[int, *Ts]",
"x: tuple[int, *tuple[str, ...]]",
# AugAssign
"v += 1",
# For
Expand Down Expand Up @@ -2316,8 +2330,14 @@ def main():
('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None)], []),
('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None)], []),
('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [])], []),
('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [])], []),
Expand All @@ -2327,6 +2347,9 @@ def main():
('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []),
('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []),
('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []),
('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []),
('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []),
Expand Down
65 changes: 62 additions & 3 deletions Lib/test/test_genericalias.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,13 @@
from queue import Queue, SimpleQueue
from weakref import WeakSet, ReferenceType, ref
import typing
from typing import Callable, Dict, List

from typing import TypeVar
from typing import TypeVar, TypeVarTuple
T = TypeVar('T')
K = TypeVar('K')
V = TypeVar('V')
Ts = TypeVarTuple('Ts')

class BaseTest(unittest.TestCase):
"""Test basics."""
Expand Down Expand Up @@ -162,6 +164,10 @@ class MyList(list):
self.assertEqual(repr(list[str]), 'list[str]')
self.assertEqual(repr(list[()]), 'list[()]')
self.assertEqual(repr(tuple[int, ...]), 'tuple[int, ...]')
self.assertEqual(repr(tuple[*Ts]), 'tuple[*Ts]')
self.assertEqual(repr(tuple[*tuple[int]]), 'tuple[*tuple[int]]')
self.assertEqual(repr(tuple[*tuple[int, str]]), 'tuple[*tuple[int, str]]')
self.assertEqual(repr(tuple[*tuple[int, ...]]), 'tuple[*tuple[int, ...]]')
self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr.<locals>.MyList[int]'))
self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr

Expand All @@ -173,8 +179,7 @@ def test_exposed_type(self):
self.assertEqual(a.__args__, (int,))
self.assertEqual(a.__parameters__, ())

def test_parameters(self):
from typing import List, Dict, Callable
def test_dict_parameters(self):
D0 = dict[str, int]
self.assertEqual(D0.__args__, (str, int))
self.assertEqual(D0.__parameters__, ())
Expand All @@ -190,6 +195,8 @@ def test_parameters(self):
D2b = dict[T, T]
self.assertEqual(D2b.__args__, (T, T))
self.assertEqual(D2b.__parameters__, (T,))

mrahtz marked this conversation as resolved.
Show resolved Hide resolved
def test_list_parameters(self):
L0 = list[str]
self.assertEqual(L0.__args__, (str,))
self.assertEqual(L0.__parameters__, ())
Expand All @@ -212,6 +219,56 @@ def test_parameters(self):
self.assertEqual(L5.__args__, (Callable[[K, V], K],))
self.assertEqual(L5.__parameters__, (K, V))

def test_tuple_parameters(self):
with self.assertRaises(TypeError):
tuple[*list[int]]

T0 = tuple[*Ts]
mrahtz marked this conversation as resolved.
Show resolved Hide resolved
self.assertEqual(T0.__args__, (*Ts,))
self.assertEqual(T0.__parameters__, (Ts,))
T1 = tuple[*tuple[int]]
self.assertEqual(T1.__args__, (*tuple[int],))
self.assertEqual(T1.__parameters__, ())
T2 = tuple[*tuple[T]]
self.assertEqual(T2.__args__, (*tuple[T],))
self.assertEqual(T2.__parameters__, (T,))
T3 = tuple[*tuple[int]]
self.assertEqual(T3.__args__, (*tuple[int],))
self.assertEqual(T3.__parameters__, ())
T4 = tuple[*tuple[int, str]]
self.assertEqual(T4.__args__, (*tuple[int, str],))
self.assertEqual(T4.__parameters__, ())
T5 = tuple[*tuple[*Ts]]
self.assertEqual(T5.__args__, (*tuple[*Ts],))
self.assertEqual(T5.__parameters__, (Ts,))
T5_0 = T5.__args__[0]
self.assertEqual(T5_0.__args__, (*Ts,))
self.assertEqual(T5_0.__parameters__, (Ts,))
T6 = tuple[*tuple[*Ts, int]]
self.assertEqual(T6.__args__, (*tuple[*Ts, int],))
self.assertEqual(T6.__parameters__, (Ts,))
T6_0 = T6.__args__[0]
self.assertEqual(T6_0.__args__, (*Ts, int))
self.assertEqual(T6_0.__parameters__, (Ts,))
T7 = tuple[*tuple[int, *Ts]]
self.assertEqual(T7.__args__, (*tuple[int, *Ts],))
self.assertEqual(T7.__parameters__, (Ts,))
T7_0 = T7.__args__[0]
self.assertEqual(T7_0.__args__, (int, *Ts))
self.assertEqual(T7_0.__parameters__, (Ts,))
T8 = tuple[*tuple[int, *Ts, str]]
self.assertEqual(T8.__args__, (*tuple[int, *Ts, str],))
self.assertEqual(T8.__parameters__, (Ts,))
T8_0 = T8.__args__[0]
self.assertEqual(T8_0.__args__, (int, *Ts, str))
self.assertEqual(T8_0.__parameters__, (Ts,))
T9 = tuple[*tuple[T, *Ts]]
self.assertEqual(T9.__args__, (*tuple[T, *Ts],))
self.assertEqual(T9.__parameters__, (T, Ts))
T10 = tuple[*tuple[*Ts, T]]
self.assertEqual(T10.__args__, (*tuple[*Ts, T],))
self.assertEqual(T10.__parameters__, (Ts, T))

def test_parameter_chaining(self):
from typing import List, Dict, Union, Callable
self.assertEqual(list[T][int], list[int])
Expand Down Expand Up @@ -242,6 +299,8 @@ def test_parameter_chaining(self):
def test_equality(self):
self.assertEqual(list[int], list[int])
self.assertEqual(dict[str, int], dict[str, int])
self.assertEqual((*tuple[int],)[0], (*tuple[int],)[0])
self.assertEqual(tuple[*tuple[int]], tuple[*tuple[int]])
self.assertNotEqual(dict[str, int], dict[str, str])
self.assertNotEqual(list, list[int])
self.assertNotEqual(list[int], list)
Expand Down
Loading