From 66ccbc8d00a1ff81f4fe72bbcfe789a429453aeb Mon Sep 17 00:00:00 2001
From: Charlie Marsh <charlie.r.marsh@gmail.com>
Date: Sun, 14 May 2023 23:03:47 -0400
Subject: [PATCH] Always emit non-logical newlines for 'empty' lines

---
 parser/src/lexer.rs | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
index 1f72667a..209df938 100644
--- a/parser/src/lexer.rs
+++ b/parser/src/lexer.rs
@@ -625,7 +625,10 @@ where
                 }
                 Some('\n' | '\r') => {
                     // Empty line!
+                    let tok_start = self.get_pos();
                     self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end)));
                     spaces = 0;
                     tabs = 0;
                 }
@@ -1121,7 +1124,7 @@ where
                         return Err(LexicalError {
                             error: LexicalErrorType::LineContinuationError,
                             location: self.get_pos(),
-                        })
+                        });
                     }
                 }
 
@@ -1501,6 +1504,7 @@ mod tests {
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                     ]
                 );
@@ -1540,10 +1544,12 @@ mod tests {
                         },
                         Tok::Colon,
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Indent,
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                         Tok::Dedent,
                     ]
@@ -1578,10 +1584,12 @@ mod tests {
                         },
                         Tok::Colon,
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Indent,
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                         Tok::Dedent,
                     ]
@@ -1699,15 +1707,15 @@ mod tests {
 
     #[test]
     fn test_logical_newline_line_comment() {
-        let source = "#Hello\n#World";
+        let source = "#Hello\n#World\n";
         let tokens = lex_source(source);
         assert_eq!(
             tokens,
             vec![
                 Tok::Comment("#Hello".to_owned()),
-                // tokenize.py does put an NL here...
+                Tok::NonLogicalNewline,
                 Tok::Comment("#World".to_owned()),
-                // ... and here, but doesn't seem very useful.
+                Tok::NonLogicalNewline,
             ]
         );
     }