diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 1f72667a..209df938 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -625,7 +625,10 @@ where } Some('\n' | '\r') => { // Empty line! + let tok_start = self.get_pos(); self.next_char(); + let tok_end = self.get_pos(); + self.emit((Tok::NonLogicalNewline, TextRange::new(tok_start, tok_end))); spaces = 0; tabs = 0; } @@ -1121,7 +1124,7 @@ where return Err(LexicalError { error: LexicalErrorType::LineContinuationError, location: self.get_pos(), - }) + }); } } @@ -1501,6 +1504,7 @@ mod tests { Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, ] ); @@ -1540,10 +1544,12 @@ mod tests { }, Tok::Colon, Tok::Newline, + Tok::NonLogicalNewline, Tok::Indent, Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, Tok::Dedent, ] @@ -1578,10 +1584,12 @@ mod tests { }, Tok::Colon, Tok::Newline, + Tok::NonLogicalNewline, Tok::Indent, Tok::Return, Tok::Int { value: BigInt::from(99) }, Tok::Newline, + Tok::NonLogicalNewline, Tok::Dedent, Tok::Dedent, ] @@ -1699,15 +1707,15 @@ mod tests { #[test] fn test_logical_newline_line_comment() { - let source = "#Hello\n#World"; + let source = "#Hello\n#World\n"; let tokens = lex_source(source); assert_eq!( tokens, vec![ Tok::Comment("#Hello".to_owned()), - // tokenize.py does put an NL here... + Tok::NonLogicalNewline, Tok::Comment("#World".to_owned()), - // ... and here, but doesn't seem very useful. + Tok::NonLogicalNewline, ] ); }