Skip to content

Commit

Permalink
Consume the escaped Windows newline (\r\n) for FStringMiddle (#7722)
Browse files Browse the repository at this point in the history
## Summary

This PR fixes a bug where if a Windows newline (`\r\n`) character was
escaped, then only the `\r` was consumed and not `\n` leading to an
unterminated string error.

## Test Plan

Add new test cases to check the newline escapes.

fixes: #7632
  • Loading branch information
dhruvmanila committed Oct 1, 2023
1 parent e72d617 commit e91ffe3
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 2 deletions.
28 changes: 26 additions & 2 deletions crates/ruff_python_parser/src/lexer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -584,7 +584,7 @@ impl<'source> Lexer<'source> {
location: self.offset(),
});
}
'\n' if !fstring.is_triple_quoted() => {
'\n' | '\r' if !fstring.is_triple_quoted() => {
return Err(LexicalError {
error: LexicalErrorType::FStringError(FStringErrorType::UnterminatedString),
location: self.offset(),
Expand All @@ -603,7 +603,11 @@ impl<'source> Lexer<'source> {
}
}
// Consume the escaped character.
self.cursor.bump();
if self.cursor.eat_char('\r') {
self.cursor.eat_char('\n');
} else {
self.cursor.bump();
}
}
quote @ ('\'' | '"') if quote == fstring.quote_char() => {
if let Some(triple_quotes) = fstring.triple_quotes() {
Expand Down Expand Up @@ -1983,6 +1987,26 @@ def f(arg=%timeit a = b):
assert_debug_snapshot!(lex_source(source));
}

fn fstring_single_quote_escape_eol(eol: &str) -> Vec<Spanned> {
let source = format!(r"f'text \{eol} more text'");
lex_source(&source)
}

#[test]
fn test_fstring_single_quote_escape_unix_eol() {
assert_debug_snapshot!(fstring_single_quote_escape_eol(UNIX_EOL));
}

#[test]
fn test_fstring_single_quote_escape_mac_eol() {
assert_debug_snapshot!(fstring_single_quote_escape_eol(MAC_EOL));
}

#[test]
fn test_fstring_single_quote_escape_windows_eol() {
assert_debug_snapshot!(fstring_single_quote_escape_eol(WINDOWS_EOL));
}

#[test]
fn test_fstring_escape() {
let source = r#"f"\{x:\"\{x}} \"\"\
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: fstring_single_quote_escape_eol(MAC_EOL)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "text \\\r more text",
is_raw: false,
},
2..19,
),
(
FStringEnd,
19..20,
),
(
Newline,
20..20,
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: fstring_single_quote_escape_eol(UNIX_EOL)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "text \\\n more text",
is_raw: false,
},
2..19,
),
(
FStringEnd,
19..20,
),
(
Newline,
20..20,
),
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
---
source: crates/ruff_python_parser/src/lexer.rs
expression: fstring_single_quote_escape_eol(WINDOWS_EOL)
---
[
(
FStringStart,
0..2,
),
(
FStringMiddle {
value: "text \\\r\n more text",
is_raw: false,
},
2..20,
),
(
FStringEnd,
20..21,
),
(
Newline,
21..21,
),
]

0 comments on commit e91ffe3

Please sign in to comment.