Don't swallow trailing whitespace

native/libcst/src/lib.rs

@@ -149,6 +149,22 @@ mod test {
         parse_module("def g(a, b): ...", None).expect("parse error");
     }
 
+    #[test]
+    fn test_single_statement_with_no_newline() {
+        for src in &[
+            "(\n \\\n)",
+            "(\n  \\\n)",
+            "(\n    '''\n''')",
+            "del _",
+            "if _:\n    '''\n)'''",
+            "if _:\n    ('''\n''')",
+            "if _:\n     '''\n  '''",
+            "if _:\n        '''\n    ''' ",
+        ] {
+            parse_module(src, None).unwrap_or_else(|e| panic!("'{}' doesn't parse: {}", src, e));
+        }
+    }
+
     #[test]
     fn bol_offset_first_line() {
         assert_eq!(0, bol_offset("hello", 1));

native/libcst/src/nodes/module.rs

@@ -77,12 +77,8 @@ impl<'r, 'a> Inflate<'a> for DeflatedModule<'r, 'a> {
                 }
             }
             if let Some(num) = last_indented {
-                if num + 1 == footer.len() {
-                    footer = vec![];
-                } else {
-                    let (_, rest) = footer.split_at(num + 1);
-                    footer = rest.to_vec();
-                }
+                let (_, rest) = footer.split_at(num);
+                footer = rest.to_vec();
             }
         } else {
             swap(&mut header, &mut footer);

native/libcst/src/tokenizer/core/mod.rs

@@ -334,10 +334,7 @@ impl<'t> TokState<'t> {
             return match self.text_pos.peek() {
                 // Check for EOF now
                 None => {
-                    if self.missing_nl_before_eof
-                        && self.text_pos.byte_column_number() != self.bol_width
-                        && !self.blank_line
-                    {
+                    if self.missing_nl_before_eof && !self.blank_line {
                         self.at_bol = true;
                         self.missing_nl_before_eof = false;
                         Ok(TokType::Newline)

native/libcst/src/tokenizer/tests.rs

@@ -718,6 +718,19 @@ fn test_fake_newline() {
     );
 }
 
+#[test]
+fn test_fake_newline_when_at_bol() {
+    assert_eq!(
+        tokenize_with_end_marker("(\n \\\n)", &default_config()),
+        Ok(vec![
+            (TokType::Op, "("),
+            (TokType::Op, ")"),
+            (TokType::Newline, ""),
+            (TokType::EndMarker, "")
+        ])
+    )
+}
+
 #[test]
 fn test_no_fake_newline_for_empty_input() {
     assert_eq!(

native/libcst/tests/fixtures/trailing_whitespace.py

@@ -0,0 +1,5 @@
+
+
+x = 42
+print(x)
+