Index multiline f-strings (#9837)

Fix #9777.
astral-sh · Feb 6, 2024 · df7fb95 · df7fb95
1 parent 83195a6
commit df7fb95
Show file tree

Hide file tree

Showing 29 changed files with 180 additions and 25 deletions.
diff --git a/crates/ruff_linter/resources/test/fixtures/pycodestyle/W291.py b/crates/ruff_linter/resources/test/fixtures/pycodestyle/W291.py
@@ -1,2 +1,15 @@
 '''trailing whitespace 
 inside a multiline string'''
+
+f'''trailing whitespace 
+inside a multiline f-string'''
+
+# Trailing whitespace after `{`
+f'abc { 
+    1 + 2
+}'
+
+# Trailing whitespace after `2`
+f'abc {
+    1 + 2 
+}'
diff --git a/crates/ruff_linter/src/rules/flake8_quotes/rules/avoidable_escaped_quote.rs b/crates/ruff_linter/src/rules/flake8_quotes/rules/avoidable_escaped_quote.rs
@@ -221,6 +221,7 @@ pub(crate) fn avoidable_escaped_quote(
             Tok::FStringMiddle {
                 value: string_contents,
                 is_raw,
+                triple_quoted: _,
             } if !is_raw => {
                 let Some(context) = fstrings.last_mut() else {
                     continue;
@@ -361,6 +362,7 @@ pub(crate) fn unnecessary_escaped_quote(
             Tok::FStringMiddle {
                 value: string_contents,
                 is_raw,
+                triple_quoted: _,
             } if !is_raw => {
                 let Some(context) = fstrings.last_mut() else {
                     continue;

diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs
@@ -67,7 +67,7 @@ pub(crate) fn invalid_escape_sequence(
     token_range: TextRange,
 ) {
     let (token_source_code, string_start_location) = match token {
-        Tok::FStringMiddle { value, is_raw } => {
+        Tok::FStringMiddle { value, is_raw, .. } => {
             if *is_raw {
                 return;
             }

diff --git a/...src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__W291_W291.py.snap b/...src/rules/pycodestyle/snapshots/ruff_linter__rules__pycodestyle__tests__W291_W291.py.snap
@@ -13,5 +13,65 @@ W291.py:1:23: W291 [*] Trailing whitespace
 1   |-'''trailing whitespace 
   1 |+'''trailing whitespace
 2 2 | inside a multiline string'''
+3 3 | 
+4 4 | f'''trailing whitespace 
+
+W291.py:4:24: W291 [*] Trailing whitespace
+  |
+2 | inside a multiline string'''
+3 | 
+4 | f'''trailing whitespace 
+  |                        ^ W291
+5 | inside a multiline f-string'''
+  |
+  = help: Remove trailing whitespace
+
+ℹ Unsafe fix
+1 1 | '''trailing whitespace 
+2 2 | inside a multiline string'''
+3 3 | 
+4   |-f'''trailing whitespace 
+  4 |+f'''trailing whitespace
+5 5 | inside a multiline f-string'''
+6 6 | 
+7 7 | # Trailing whitespace after `{`
+
+W291.py:8:8: W291 [*] Trailing whitespace
+   |
+ 7 | # Trailing whitespace after `{`
+ 8 | f'abc { 
+   |        ^ W291
+ 9 |     1 + 2
+10 | }'
+   |
+   = help: Remove trailing whitespace
+
+ℹ Safe fix
+5 5 | inside a multiline f-string'''
+6 6 | 
+7 7 | # Trailing whitespace after `{`
+8   |-f'abc { 
+  8 |+f'abc {
+9 9 |     1 + 2
+10 10 | }'
+11 11 | 
+
+W291.py:14:10: W291 [*] Trailing whitespace
+   |
+12 | # Trailing whitespace after `2`
+13 | f'abc {
+14 |     1 + 2 
+   |          ^ W291
+15 | }'
+   |
+   = help: Remove trailing whitespace
+
+ℹ Safe fix
+11 11 | 
+12 12 | # Trailing whitespace after `2`
+13 13 | f'abc {
+14    |-    1 + 2 
+   14 |+    1 + 2
+15 15 | }'
 
 
diff --git a/crates/ruff_python_index/src/multiline_ranges.rs b/crates/ruff_python_index/src/multiline_ranges.rs
@@ -46,7 +46,8 @@ pub(crate) struct MultilineRangesBuilder {
 
 impl MultilineRangesBuilder {
     pub(crate) fn visit_token(&mut self, token: &Tok, range: TextRange) {
-        if let Tok::String { triple_quoted, .. } = token {
+        if let Tok::String { triple_quoted, .. } | Tok::FStringMiddle { triple_quoted, .. } = token
+        {
             if *triple_quoted {
                 self.ranges.push(range);
             }

diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs
@@ -675,6 +675,7 @@ impl<'source> Lexer<'source> {
         Ok(Some(Tok::FStringMiddle {
             value,
             is_raw: fstring.is_raw_string(),
+            triple_quoted: fstring.is_triple_quoted(),
         }))
     }
 

diff --git a/crates/ruff_python_parser/src/python.lalrpop b/crates/ruff_python_parser/src/python.lalrpop
@@ -1622,7 +1622,7 @@ FStringExpr: StringType = {
 FStringMiddlePattern: ast::FStringElement = {
     FStringReplacementField,
     <location:@L> <fstring_middle:fstring_middle> <end_location:@R> =>? {
-        let (source, is_raw) = fstring_middle;
+        let (source, is_raw, _) = fstring_middle;
         Ok(parse_fstring_literal_element(&source, is_raw, (location..end_location).into())?)
     }
 };
@@ -2067,7 +2067,8 @@ extern {
         },
         fstring_middle => token::Tok::FStringMiddle {
             value: <String>,
-            is_raw: <bool>
+            is_raw: <bool>,
+            triple_quoted: <bool>
         },
         name => token::Tok::Name { name: <String> },
         ipy_escape_command => token::Tok::IpyEscapeCommand {

diff --git a/crates/ruff_python_parser/src/python.rs b/crates/ruff_python_parser/src/python.rs
@@ -1,5 +1,5 @@
 // auto-generated: "lalrpop 0.20.0"
-// sha3: 031689e389556292d9dbd8a1b1ff8ca29bac76d83f1b345630481d620b89e1c2
+// sha3: aa0540221d25f4eadfc9e043fb4fc631d537b672b8a96785dfec2407e0524b79
 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize};
 use ruff_python_ast::{self as ast, Int, IpyEscapeKind};
 use crate::{
@@ -50,7 +50,7 @@ mod __parse__Top {
         Variant0(token::Tok),
         Variant1((f64, f64)),
         Variant2(f64),
-        Variant3((String, bool)),
+        Variant3((String, bool, bool)),
         Variant4(Int),
         Variant5((IpyEscapeKind, String)),
         Variant6(String),
@@ -151,7 +151,7 @@ mod __parse__Top {
         Variant101(ast::TypeParams),
         Variant102(core::option::Option<ast::TypeParams>),
         Variant103(ast::UnaryOp),
-        Variant104(core::option::Option<(String, bool)>),
+        Variant104(core::option::Option<(String, bool, bool)>),
     }
     const __ACTION: &[i16] = &[
         // State 0
@@ -6090,7 +6090,7 @@ mod __parse__Top {
             token::Tok::StartModule if true => Some(93),
             token::Tok::Complex { real: _, imag: _ } if true => Some(94),
             token::Tok::Float { value: _ } if true => Some(95),
-            token::Tok::FStringMiddle { value: _, is_raw: _ } if true => Some(96),
+            token::Tok::FStringMiddle { value: _, is_raw: _, triple_quoted: _ } if true => Some(96),
             token::Tok::Int { value: _ } if true => Some(97),
             token::Tok::IpyEscapeCommand { kind: _, value: _ } if true => Some(98),
             token::Tok::Name { name: _ } if true => Some(99),
@@ -6116,7 +6116,7 @@ mod __parse__Top {
                 _ => unreachable!(),
             },
             96 => match __token {
-                token::Tok::FStringMiddle { value: __tok0, is_raw: __tok1 } if true => __Symbol::Variant3((__tok0, __tok1)),
+                token::Tok::FStringMiddle { value: __tok0, is_raw: __tok1, triple_quoted: __tok2 } if true => __Symbol::Variant3((__tok0, __tok1, __tok2)),
                 _ => unreachable!(),
             },
             97 => match __token {
@@ -18386,7 +18386,7 @@ mod __parse__Top {
     fn __pop_Variant3<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
-    ) -> (TextSize, (String, bool), TextSize)
+    ) -> (TextSize, (String, bool, bool), TextSize)
      {
         match __symbols.pop() {
             Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r),
@@ -19136,7 +19136,7 @@ mod __parse__Top {
     fn __pop_Variant104<
     >(
         __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)>
-    ) -> (TextSize, core::option::Option<(String, bool)>, TextSize)
+    ) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize)
      {
         match __symbols.pop() {
             Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r),
@@ -35910,7 +35910,7 @@ fn __action184<
     (_, parameters, _): (TextSize, core::option::Option<ast::Parameters>, TextSize),
     (_, end_location_args, _): (TextSize, TextSize, TextSize),
     (_, _, _): (TextSize, token::Tok, TextSize),
-    (_, fstring_middle, _): (TextSize, core::option::Option<(String, bool)>, TextSize),
+    (_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
     (_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize),
     (_, end_location, _): (TextSize, TextSize, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@@ -36413,12 +36413,12 @@ fn __action220<
     source_code: &str,
     mode: Mode,
     (_, location, _): (TextSize, TextSize, TextSize),
-    (_, fstring_middle, _): (TextSize, (String, bool), TextSize),
+    (_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize),
     (_, end_location, _): (TextSize, TextSize, TextSize),
 ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     {
-        let (source, is_raw) = fstring_middle;
+        let (source, is_raw, _) = fstring_middle;
         Ok(parse_fstring_literal_element(&source, is_raw, (location..end_location).into())?)
     }
 }
@@ -37357,8 +37357,8 @@ fn __action281<
 >(
     source_code: &str,
     mode: Mode,
-    (_, __0, _): (TextSize, (String, bool), TextSize),
-) -> core::option::Option<(String, bool)>
+    (_, __0, _): (TextSize, (String, bool, bool), TextSize),
+) -> core::option::Option<(String, bool, bool)>
 {
     Some(__0)
 }
@@ -37371,7 +37371,7 @@ fn __action282<
     mode: Mode,
     __lookbehind: &TextSize,
     __lookahead: &TextSize,
-) -> core::option::Option<(String, bool)>
+) -> core::option::Option<(String, bool, bool)>
 {
     None
 }
@@ -48505,7 +48505,7 @@ fn __action804<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, bool), TextSize),
+    __0: (TextSize, (String, bool, bool), TextSize),
     __1: (TextSize, TextSize, TextSize),
 ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -49609,7 +49609,7 @@ fn __action839<
     __1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
     __2: (TextSize, TextSize, TextSize),
     __3: (TextSize, token::Tok, TextSize),
-    __4: (TextSize, core::option::Option<(String, bool)>, TextSize),
+    __4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
     __5: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
     __6: (TextSize, TextSize, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
@@ -64527,7 +64527,7 @@ fn __action1315<
 >(
     source_code: &str,
     mode: Mode,
-    __0: (TextSize, (String, bool), TextSize),
+    __0: (TextSize, (String, bool, bool), TextSize),
 ) -> Result<ast::FStringElement,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
     let __start0 = __0.2;
@@ -65430,7 +65430,7 @@ fn __action1347<
     __0: (TextSize, token::Tok, TextSize),
     __1: (TextSize, core::option::Option<ast::Parameters>, TextSize),
     __2: (TextSize, token::Tok, TextSize),
-    __3: (TextSize, core::option::Option<(String, bool)>, TextSize),
+    __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
     __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -77662,7 +77662,7 @@ fn __action1727<
     __0: (TextSize, token::Tok, TextSize),
     __1: (TextSize, ast::Parameters, TextSize),
     __2: (TextSize, token::Tok, TextSize),
-    __3: (TextSize, core::option::Option<(String, bool)>, TextSize),
+    __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
     __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -77693,7 +77693,7 @@ fn __action1728<
     mode: Mode,
     __0: (TextSize, token::Tok, TextSize),
     __1: (TextSize, token::Tok, TextSize),
-    __2: (TextSize, core::option::Option<(String, bool)>, TextSize),
+    __2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize),
     __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -79598,7 +79598,7 @@ fn __action1785<
     __0: (TextSize, token::Tok, TextSize),
     __1: (TextSize, ast::Parameters, TextSize),
     __2: (TextSize, token::Tok, TextSize),
-    __3: (TextSize, (String, bool), TextSize),
+    __3: (TextSize, (String, bool, bool), TextSize),
     __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {
@@ -79661,7 +79661,7 @@ fn __action1787<
     mode: Mode,
     __0: (TextSize, token::Tok, TextSize),
     __1: (TextSize, token::Tok, TextSize),
-    __2: (TextSize, (String, bool), TextSize),
+    __2: (TextSize, (String, bool, bool), TextSize),
     __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize),
 ) -> Result<crate::parser::ParenthesizedExpr,__lalrpop_util::ParseError<TextSize,token::Tok,LexicalError>>
 {

diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring.snap
@@ -11,6 +11,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: "normal ",
             is_raw: false,
+            triple_quoted: false,
         },
         2..9,
     ),
@@ -32,6 +33,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: " {another} ",
             is_raw: false,
+            triple_quoted: false,
         },
         14..27,
     ),
@@ -53,6 +55,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: " {",
             is_raw: false,
+            triple_quoted: false,
         },
         32..35,
     ),
@@ -74,6 +77,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: "}",
             is_raw: false,
+            triple_quoted: false,
         },
         42..44,
     ),

diff --git a/.../ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap b/.../ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__fstring_comments.snap
@@ -11,6 +11,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: "\n# not a comment ",
             is_raw: false,
+            triple_quoted: true,
         },
         4..21,
     ),
@@ -46,6 +47,7 @@ expression: lex_source(source)
         FStringMiddle {
             value: " # not a comment\n",
             is_raw: false,
+            triple_quoted: true,
         },
         42..59,
     ),
-Original file line number
+Diff line change
@@ Expand Up / @@ -675,6 +675,7 @@ impl<'source> Lexer<'source> { @@
             Ok(Some(Tok::FStringMiddle {
                 value,
                 is_raw: fstring.is_raw_string(),
+                triple_quoted: fstring.is_triple_quoted(),
             }))
         }
@@ Expand Down @@