From c371cc9c2afba5600f8749c4ebcd93b4c9dc6a4e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 29 Jun 2024 10:58:42 +0100 Subject: [PATCH 01/10] gh-121130: Fix f-string format specifiers with debug expressions --- Lib/test/test_fstring.py | 6 ++++++ .../2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst | 2 ++ Parser/lexer/lexer.c | 14 ++++++++++++-- Parser/lexer/state.c | 1 + Parser/lexer/state.h | 1 + 5 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 49c6f761e5b4f0..558af76ddf4b66 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1601,6 +1601,10 @@ def f(a): self.assertEqual(f'{f(a=4)}', '3=') self.assertEqual(x, 4) + # Check debug expressions in format spec + y = 20 + self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + # Make sure __format__ is being called. class C: def __format__(self, s): @@ -1614,9 +1618,11 @@ def __repr__(self): self.assertEqual(f'{C()=: }', 'C()=FORMAT- ') self.assertEqual(f'{C()=:x}', 'C()=FORMAT-x') self.assertEqual(f'{C()=!r:*^20}', 'C()=********REPR********') + self.assertEqual(f"{C():{20=}}", 'FORMAT-20=20') self.assertRaises(SyntaxError, eval, "f'{C=]'") + # Make sure leading and following text works. x = 'foo' self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst new file mode 100644 index 00000000000000..7084f0cbebbb73 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-29-10-46-14.gh-issue-121130.Rj66Xs.rst @@ -0,0 +1,2 @@ +Fix f-strings with debug expressions in format specifiers. Patch by Pablo +Galindo diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 82b0e4ee352d62..5366ab949e4483 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -989,6 +989,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t the_current_tok->last_expr_buffer = NULL; the_current_tok->last_expr_size = 0; the_current_tok->last_expr_end = -1; + the_current_tok->in_format_spec = 0; the_current_tok->f_string_debug = 0; switch (*tok->start) { @@ -1137,15 +1138,20 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t * by the `{` case, so for ensuring that we are on the 0th level, we need * to adjust it manually */ int cursor = current_tok->curly_bracket_depth - (c != '{'); - if (cursor == 0 && !_PyLexer_update_fstring_expr(tok, c)) { + int in_format_spec = current_tok->in_format_spec; + int cursor_in_format_with_debug = + cursor == 1 && (current_tok->f_string_debug || in_format_spec); + int cursor_valid = cursor == 0 || cursor_in_format_with_debug; + if (cursor_valid && !_PyLexer_update_fstring_expr(tok, c)) { return MAKE_TOKEN(ENDMARKER); } - if (cursor == 0 && c != '{' && set_fstring_expr(tok, token, c)) { + if (cursor_valid && c != '{' && set_fstring_expr(tok, token, c)) { return MAKE_TOKEN(ERRORTOKEN); } if (c == ':' && cursor == current_tok->curly_bracket_expr_start_depth) { current_tok->kind = TOK_FSTRING_MODE; + current_tok->in_format_spec = 1; p_start = tok->start; p_end = tok->cur; return MAKE_TOKEN(_PyToken_OneChar(c)); @@ -1235,6 +1241,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (c == '}' && current_tok->curly_bracket_depth == current_tok->curly_bracket_expr_start_depth) { current_tok->curly_bracket_expr_start_depth--; current_tok->kind = TOK_FSTRING_MODE; + current_tok->in_format_spec = 0; current_tok->f_string_debug = 0; } } @@ -1337,6 +1344,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct if (in_format_spec && c == '\n') { tok_backup(tok, c); TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; return MAKE_TOKEN(FSTRING_MIDDLE); @@ -1387,6 +1395,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply")); } TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; } else { @@ -1413,6 +1422,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok_backup(tok, peek); tok_backup(tok, c); TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; + current_tok->in_format_spec = 0; p_start = tok->start; p_end = tok->cur; } diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 653ddafd411095..647f291911564c 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -74,6 +74,7 @@ free_fstring_expressions(struct tok_state *tok) mode->last_expr_buffer = NULL; mode->last_expr_size = 0; mode->last_expr_end = -1; + mode->in_format_spec = 0; } } } diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 61d090d6d2fe21..9ed3babfdbfbf1 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -58,6 +58,7 @@ typedef struct _tokenizer_mode { Py_ssize_t last_expr_end; char* last_expr_buffer; int f_string_debug; + int in_format_spec; } tokenizer_mode; /* Tokenizer state */ From 89bd65ac7150763654f2ea5766aca4dfbfbe4850 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 29 Jun 2024 11:56:13 +0100 Subject: [PATCH 02/10] Fix unparsing of fstring with format specs --- Parser/action_helpers.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 91b7e2f1058423..03876ebfffc5a3 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1444,8 +1444,16 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re conversion_val = (int)'r'; } + expr_ty format_expr = format ? (expr_ty) format->result : NULL; + if (format_expr && format_expr->kind == JoinedStr_kind && asdl_seq_LEN(format_expr->v.JoinedStr.values) == 1) { + expr_ty format_value = asdl_seq_GET(format_expr->v.JoinedStr.values, 0); + if (format_value->kind == JoinedStr_kind) { + format_expr = format_value; + } + } + expr_ty formatted_value = _PyAST_FormattedValue( - expression, conversion_val, format ? (expr_ty) format->result : NULL, + expression, conversion_val, format_expr, lineno, col_offset, end_lineno, end_col_offset, arena ); From f4aed72fcf4af777f77b79f72f06075bf4801610 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 1 Jul 2024 21:44:44 +0100 Subject: [PATCH 03/10] Fixes --- Lib/test/test_fstring.py | 3 ++ Parser/action_helpers.c | 75 +++++++++++++++++++++++----------------- Parser/lexer/lexer.c | 30 ++++++++++------ 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 558af76ddf4b66..50fcb4708e3741 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -8,6 +8,7 @@ # Unicode identifiers in tests is allowed by PEP 3131. import ast +import datetime import dis import os import re @@ -1604,6 +1605,8 @@ def f(a): # Check debug expressions in format spec y = 20 self.assertEqual(f"{2:{y=}}", "yyyyyyyyyyyyyyyyyyy2") + self.assertEqual(f"{datetime.datetime.now():h1{y=}h2{y=}h3{y=}}", + 'h1y=20h2y=20h3y=20') # Make sure __format__ is being called. class C: diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 03876ebfffc5a3..92f3ac3eecdb53 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -967,6 +967,8 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) return result_token_with_metadata(p, conv, conv_token->metadata); } +static asdl_expr_seq * +unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions); ResultTokenWithMetadata * _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) @@ -1000,13 +1002,20 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { continue; } - asdl_seq_SET(resized_spec, j++, item); + asdl_seq_SET(resized_spec, j++, item); } assert(j == non_empty_count); spec = resized_spec; } - expr_ty res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, - end_col_offset, p->arena); + expr_ty res; + if (asdl_seq_LEN(spec) == 0) { + res = _PyAST_JoinedStr(spec, lineno, col_offset, end_lineno, + end_col_offset, p->arena); + } else { + res = _PyPegen_concatenate_strings(p, spec, + lineno, col_offset, end_lineno, + end_col_offset, arena); + } if (!res) { return NULL; } @@ -1306,6 +1315,7 @@ unpack_top_level_joined_strs(Parser *p, asdl_expr_seq *raw_expressions) expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) { + asdl_expr_seq *expr = unpack_top_level_joined_strs(p, raw_expressions); Py_ssize_t n_items = asdl_seq_LEN(expr); @@ -1444,16 +1454,8 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re conversion_val = (int)'r'; } - expr_ty format_expr = format ? (expr_ty) format->result : NULL; - if (format_expr && format_expr->kind == JoinedStr_kind && asdl_seq_LEN(format_expr->v.JoinedStr.values) == 1) { - expr_ty format_value = asdl_seq_GET(format_expr->v.JoinedStr.values, 0); - if (format_value->kind == JoinedStr_kind) { - format_expr = format_value; - } - } - expr_ty formatted_value = _PyAST_FormattedValue( - expression, conversion_val, format_expr, + expression, conversion_val, format ? (expr_ty) format->result : NULL, lineno, col_offset, end_lineno, end_col_offset, arena ); @@ -1478,7 +1480,6 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re debug_end_offset = end_col_offset; debug_metadata = closing_brace->metadata; } - expr_ty debug_text = _PyAST_Constant(debug_metadata, NULL, lineno, col_offset + 1, debug_end_line, debug_end_offset - 1, p->arena); if (!debug_text) { @@ -1511,16 +1512,23 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t n_flattened_elements = 0; for (i = 0; i < len; i++) { expr_ty elem = asdl_seq_GET(strings, i); - if (elem->kind == Constant_kind) { - if (PyBytes_CheckExact(elem->v.Constant.value)) { - bytes_found = 1; - } else { - unicode_string_found = 1; - } - n_flattened_elements++; - } else { - n_flattened_elements += asdl_seq_LEN(elem->v.JoinedStr.values); - f_string_found = 1; + switch(elem->kind) { + case Constant_kind: + if (PyBytes_CheckExact(elem->v.Constant.value)) { + bytes_found = 1; + } else { + unicode_string_found = 1; + } + n_flattened_elements++; + break; + case JoinedStr_kind: + n_flattened_elements += asdl_seq_LEN(elem->v.JoinedStr.values); + f_string_found = 1; + break; + default: + n_flattened_elements++; + f_string_found = 1; + break; } } @@ -1562,16 +1570,19 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t j = 0; for (i = 0; i < len; i++) { expr_ty elem = asdl_seq_GET(strings, i); - if (elem->kind == Constant_kind) { - asdl_seq_SET(flattened, current_pos++, elem); - } else { - for (j = 0; j < asdl_seq_LEN(elem->v.JoinedStr.values); j++) { - expr_ty subvalue = asdl_seq_GET(elem->v.JoinedStr.values, j); - if (subvalue == NULL) { - return NULL; + switch(elem->kind) { + case JoinedStr_kind: + for (j = 0; j < asdl_seq_LEN(elem->v.JoinedStr.values); j++) { + expr_ty subvalue = asdl_seq_GET(elem->v.JoinedStr.values, j); + if (subvalue == NULL) { + return NULL; + } + asdl_seq_SET(flattened, current_pos++, subvalue); } - asdl_seq_SET(flattened, current_pos++, subvalue); - } + break; + default: + asdl_seq_SET(flattened, current_pos++, elem); + break; } } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 5366ab949e4483..63954364ce7c7b 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -212,10 +212,15 @@ _PyLexer_update_fstring_expr(struct tok_state *tok, char cur) case '}': case '!': case ':': - if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); + { + Py_ssize_t size = strlen(tok->start); + if (tok_mode->last_expr_size - size > 0) { + tok_mode->last_expr_end = size; + } else { + tok_mode->last_expr_end = tok_mode->last_expr_size; + } + break; } - break; default: Py_UNREACHABLE(); } @@ -1139,13 +1144,13 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t * to adjust it manually */ int cursor = current_tok->curly_bracket_depth - (c != '{'); int in_format_spec = current_tok->in_format_spec; - int cursor_in_format_with_debug = - cursor == 1 && (current_tok->f_string_debug || in_format_spec); - int cursor_valid = cursor == 0 || cursor_in_format_with_debug; - if (cursor_valid && !_PyLexer_update_fstring_expr(tok, c)) { + int cursor_in_format_with_debug = + cursor == 1 && (current_tok->f_string_debug || in_format_spec); + int cursor_valid = cursor == 0 || cursor_in_format_with_debug; + if ((cursor_valid) && !_PyLexer_update_fstring_expr(tok, c)) { return MAKE_TOKEN(ENDMARKER); } - if (cursor_valid && c != '{' && set_fstring_expr(tok, token, c)) { + if ((cursor_valid) && c != '{' && set_fstring_expr(tok, token, c)) { return MAKE_TOKEN(ERRORTOKEN); } @@ -1328,11 +1333,15 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN(ERRORTOKEN); } int in_format_spec = ( - current_tok->last_expr_end != -1 + current_tok->in_format_spec && INSIDE_FSTRING_EXPR(current_tok) ); + if (!_PyLexer_update_fstring_expr(tok, '{')) { + return MAKE_TOKEN(ENDMARKER); + } + if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); @@ -1415,7 +1424,8 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct // scanning (indicated by the end of the expression being set) and we are not at the top level // of the bracket stack (-1 is the top level). Since format specifiers can't legally use double // brackets, we can bypass it here. - if (peek == '}' && !in_format_spec) { + int cursor = current_tok->curly_bracket_depth; + if (peek == '}' && !in_format_spec && cursor == 0) { p_start = tok->start; p_end = tok->cur - 1; } else { From 93ab4a6b3d3c69e15c2eb2837d0da3fe7562b735 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 1 Jul 2024 21:50:04 +0100 Subject: [PATCH 04/10] Fixes --- Lib/test/test_ast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index d8a5caf2112a4b..593e74b2889973 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3269,7 +3269,7 @@ def main(): ('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), ('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), ('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), -('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('Constant', (1, 5, 1, 8), '.2f', None))])), ('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), ('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), ] From b97f4fd55955263a0d4ada6eb7b456f7eb4db8db Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 2 Jul 2024 13:11:12 +0100 Subject: [PATCH 05/10] Fixes --- Parser/lexer/lexer.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 63954364ce7c7b..699356a77704ba 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -1329,7 +1329,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok->multi_line_start = tok->line_start; while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); - if (tok->done == E_ERROR) { + if (tok->done == E_ERROR || tok->done == E_DECODE) { return MAKE_TOKEN(ERRORTOKEN); } int in_format_spec = ( @@ -1338,10 +1338,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct INSIDE_FSTRING_EXPR(current_tok) ); - if (!_PyLexer_update_fstring_expr(tok, '{')) { - return MAKE_TOKEN(ENDMARKER); - } - + if (!_PyLexer_update_fstring_expr(tok, '{')) { + return MAKE_TOKEN(ENDMARKER); + } if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); From 3333a492ece34067b0218ede55ef45f86f1886f5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 2 Jul 2024 18:57:52 +0100 Subject: [PATCH 06/10] fix tests --- Doc/library/ast.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index f7e8afa7000392..56662615275228 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -316,9 +316,7 @@ Literals args=[ Name(id='a', ctx=Load())]), conversion=-1, - format_spec=JoinedStr( - values=[ - Constant(value='.3')]))])) + format_spec=Constant(value='.3'))])) .. class:: List(elts, ctx) From 899cf9a245454fddb6f72d7fb331a7993734d9c1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 2 Jul 2024 19:11:57 +0100 Subject: [PATCH 07/10] fix tests --- Parser/lexer/lexer.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 699356a77704ba..83180f5eb25b62 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -212,15 +212,10 @@ _PyLexer_update_fstring_expr(struct tok_state *tok, char cur) case '}': case '!': case ':': - { - Py_ssize_t size = strlen(tok->start); - if (tok_mode->last_expr_size - size > 0) { - tok_mode->last_expr_end = size; - } else { - tok_mode->last_expr_end = tok_mode->last_expr_size; - } - break; - } + if (tok_mode->last_expr_end == -1) { + tok_mode->last_expr_end = strlen(tok->start); + } + break; default: Py_UNREACHABLE(); } @@ -1338,9 +1333,6 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct INSIDE_FSTRING_EXPR(current_tok) ); - if (!_PyLexer_update_fstring_expr(tok, '{')) { - return MAKE_TOKEN(ENDMARKER); - } if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); @@ -1394,6 +1386,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct } if (c == '{') { + if (!_PyLexer_update_fstring_expr(tok, c)) { + return MAKE_TOKEN(ENDMARKER); + } int peek = tok_nextc(tok); if (peek != '{' || in_format_spec) { tok_backup(tok, peek); From 4114e0cc11fb89ee65e135b3c3cb8fa8da225d18 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 3 Jul 2024 09:55:42 +0100 Subject: [PATCH 08/10] Update Parser/lexer/lexer.c --- Parser/lexer/lexer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 83180f5eb25b62..1c2a6750ed47f9 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -213,8 +213,8 @@ _PyLexer_update_fstring_expr(struct tok_state *tok, char cur) case '!': case ':': if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); - } + ok_mode->last_expr_end = strlen(tok->start); + } break; default: Py_UNREACHABLE(); From 88a02968d2ebd2bd8d13e107547e80a6d89a3dc6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 4 Jul 2024 15:32:41 +0100 Subject: [PATCH 09/10] Update Parser/lexer/lexer.c Co-authored-by: Lysandros Nikolaou --- Parser/lexer/lexer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 1c2a6750ed47f9..93b5fbd34a2a68 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -213,7 +213,7 @@ _PyLexer_update_fstring_expr(struct tok_state *tok, char cur) case '!': case ':': if (tok_mode->last_expr_end == -1) { - ok_mode->last_expr_end = strlen(tok->start); + tok_mode->last_expr_end = strlen(tok->start); } break; default: From e6e0f265150c9da7780d3984d9a2af712b72650d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 4 Jul 2024 18:18:26 +0100 Subject: [PATCH 10/10] Update action_helpers.c Co-authored-by: Lysandros Nikolaou --- Parser/action_helpers.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 92f3ac3eecdb53..149e179dab118a 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1002,7 +1002,7 @@ _PyPegen_setup_full_format_spec(Parser *p, Token *colon, asdl_expr_seq *spec, in PyUnicode_GET_LENGTH(item->v.Constant.value) == 0) { continue; } - asdl_seq_SET(resized_spec, j++, item); + asdl_seq_SET(resized_spec, j++, item); } assert(j == non_empty_count); spec = resized_spec;