fix: Recognize raw string literals without any #

Fixes gluon-lang#885
Marwes · Oct 15, 2020 · 4d66fbb · 4d66fbb
1 parent c313ab8
commit 4d66fbb
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 9 deletions.
diff --git a/parser/src/token.rs b/parser/src/token.rs
@@ -581,12 +581,7 @@ impl<'input> Tokenizer<'input> {
             match self.bump() {
                 Some((_, b'"')) => {
                     let mut found_delimiters = 0;
-                    while let Some((_, ch)) = self.bump() {
-                        match ch {
-                            b'#' => found_delimiters += 1,
-                            b'"' => found_delimiters = 0,
-                            _ => break,
-                        }
+                    loop {
                         if found_delimiters == delimiters {
                             let end = self.next_loc();
                             let mut content_end = end;
@@ -596,6 +591,15 @@ impl<'input> Tokenizer<'input> {
                             let token = Token::StringLiteral(StringLiteral::Raw(string));
                             return Ok(pos::spanned2(start, end, token));
                         }
+                        if let Some((_, ch)) = self.bump() {
+                            match ch {
+                                b'#' => found_delimiters += 1,
+                                b'"' => found_delimiters = 0,
+                                _ => break,
+                            }
+                        } else {
+                            break;
+                        }
                     }
                 }
                 _ => break,
@@ -1053,7 +1057,7 @@ mod test {
     #[test]
     fn raw_string_literals() {
         test(
-            r#########"foo r#"bar" "# baz r##""## "#########,
+            r#########"foo r#"bar" "# baz r##""## r#"""#  "#########,
             vec![
                 (r####"~~~                      "####, Identifier("foo")),
                 (
@@ -1065,6 +1069,10 @@ mod test {
                     r#"                   ~~~~~~~     "#,
                     Token::StringLiteral(StringLiteral::Raw("")),
                 ),
+                (
+                    r#"                           ~~~~~~     "#,
+                    Token::StringLiteral(StringLiteral::Raw("\"")),
+                ),
             ],
         );
     }

diff --git a/parser/tests/basic.rs b/parser/tests/basic.rs
@@ -360,6 +360,14 @@ test_parse! {
     }
 }
 
+test_parse! {
+    raw_string_literal,
+    r##"  r"asdf" "##,
+    |_arena| {
+        string("asdf")
+    }
+}
+
 #[test]
 fn span_identifier() {
     let _ = ::env_logger::try_init();

diff --git a/parser/tests/support/mod.rs b/parser/tests/support/mod.rs
@@ -6,8 +6,8 @@ use crate::base::{
     ast::{
         self, walk_mut_alias, walk_mut_ast_type, walk_mut_expr, walk_mut_pattern, Alternative,
         Argument, Array, AstType, DisplayEnv, Do, Expr, ExprField, IdentEnv, Lambda, Literal,
-        MutVisitor, Pattern, RootExpr, SpannedAlias, SpannedAstType, SpannedExpr, SpannedIdent,
-        SpannedPattern, TypeBinding, TypedIdent, ValueBinding,Sp,
+        MutVisitor, Pattern, RootExpr, Sp, SpannedAlias, SpannedAstType, SpannedExpr, SpannedIdent,
+        SpannedPattern, TypeBinding, TypedIdent, ValueBinding,
     },
     error::Errors,
     kind::Kind,
@@ -548,6 +548,7 @@ macro_rules! test_parse {
             let text = $text;
             let e = parse_clear_span!(text);
             mk_ast_arena!(arena);
+            let _: &Arena<String> = &*arena;
             fn call<A, R>(a: A, f: impl FnOnce(A) -> R) -> R {
                 f(a)
             }