Always emit non-logical newlines for 'empty' lines

RustPython · May 15, 2023 · 66ccbc8 · 66ccbc8
1 parent a983f43
commit 66ccbc8
Showing 1 changed file with 12 additions and 4 deletions.
diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs
@@ -625,7 +625,10 @@ where
                 }
                 Some('\n' | '\r') => {
                     // Empty line!
+                    let tok_start = self.get_pos();
                     self.next_char();
+                    let tok_end = self.get_pos();
+                    self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end)));
                     spaces = 0;
                     tabs = 0;
                 }
@@ -1121,7 +1124,7 @@ where
                         return Err(LexicalError {
                             error: LexicalErrorType::LineContinuationError,
                             location: self.get_pos(),
-                        })
+                        });
                     }
                 }
 
@@ -1501,6 +1504,7 @@ mod tests {
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                     ]
                 );
@@ -1540,10 +1544,12 @@ mod tests {
                         },
                         Tok::Colon,
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Indent,
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                         Tok::Dedent,
                     ]
@@ -1578,10 +1584,12 @@ mod tests {
                         },
                         Tok::Colon,
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Indent,
                         Tok::Return,
                         Tok::Int { value: BigInt::from(99) },
                         Tok::Newline,
+                        Tok::NonLogicalNewline,
                         Tok::Dedent,
                         Tok::Dedent,
                     ]
@@ -1699,15 +1707,15 @@ mod tests {
 
     #[test]
     fn test_logical_newline_line_comment() {
-        let source = "#Hello\n#World";
+        let source = "#Hello\n#World\n";
         let tokens = lex_source(source);
         assert_eq!(
             tokens,
             vec![
                 Tok::Comment("#Hello".to_owned()),
-                // tokenize.py does put an NL here...
+                Tok::NonLogicalNewline,
                 Tok::Comment("#World".to_owned()),
-                // ... and here, but doesn't seem very useful.
+                Tok::NonLogicalNewline,
             ]
         );
     }