Spans should end before CRLF, not in the middle of it

projectfluent · Oct 18, 2018 · e5cf0c0 · e5cf0c0
1 parent 61187c0
commit e5cf0c0
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 51 deletions.
diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js
@@ -482,7 +482,7 @@ export default class FluentParser {
         return new AST.TextElement(buffer);
       }
 
-      if (ps.currentChar === EOL) {
+      if (ch === EOL) {
         if (!ps.isNextLineValue({skip: false})) {
           return new AST.TextElement(buffer);
         }

diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js
@@ -10,32 +10,45 @@ export class ParserStream {
     this.peekOffset = 0;
   }
 
+  charAt(offset) {
+    // When the cursor is at CRLF, return LF but don't move the cursor.
+    // The cursor still points to the EOL position, which in this case is the
+    // beginning of the compound CRLF sequence. This ensures slices of
+    // [inclusive, exclusive) continue to work properly.
+    if (this.string[offset] === "\r"
+        && this.string[offset + 1] === "\n") {
+      return "\n";
+    }
+
+    return this.string[offset];
+  }
+
   get currentChar() {
-    return this.string[this.index];
+    return this.charAt(this.index);
   }
 
   get currentPeek() {
-    return this.string[this.index + this.peekOffset];
+    return this.charAt(this.index + this.peekOffset);
   }
 
   next() {
     this.peekOffset = 0;
-    this.index++;
-    // Normalize CRLF to LF.
+    // Skip over the CRLF as if it was a single character.
     if (this.string[this.index] === "\r"
         && this.string[this.index + 1] === "\n") {
       this.index++;
     }
+    this.index++;
     return this.string[this.index];
   }
 
   peek() {
-    this.peekOffset++;
-    // Normalize CRLF to LF.
+    // Skip over the CRLF as if it was a single character.
     if (this.string[this.index + this.peekOffset] === "\r"
         && this.string[this.index + this.peekOffset + 1] === "\n") {
       this.peekOffset++;
     }
+    this.peekOffset++;
     return this.string[this.index + this.peekOffset];
   }
 

diff --git a/fluent-syntax/test/fixtures_structure/crlf.ftl b/fluent-syntax/test/fixtures_structure/crlf.ftl
@@ -1,10 +1,14 @@
+
 key01 = Value 01
 key02 =
+
     Value 02
     Continued
 
-# ERROR Missing value or attributes
-err03
+    .title = Title
+
+# ERROR Unclosed StringLiteral
+err03 = { "str
 
 # ERROR Missing newline after ->.
 err04 = { $sel -> }
diff --git a/fluent-syntax/test/fixtures_structure/crlf.json b/fluent-syntax/test/fixtures_structure/crlf.json
@@ -8,8 +8,8 @@
         "name": "key01",
         "span": {
           "type": "Span",
-          "start": 0,
-          "end": 5
+          "start": 2,
+          "end": 7
         }
       },
       "value": {
@@ -20,23 +20,23 @@
             "value": "Value 01",
             "span": {
               "type": "Span",
-              "start": 8,
-              "end": 17
+              "start": 10,
+              "end": 18
             }
           }
         ],
         "span": {
           "type": "Span",
-          "start": 8,
-          "end": 17
+          "start": 10,
+          "end": 18
         }
       },
       "attributes": [],
       "comment": null,
       "span": {
         "type": "Span",
-        "start": 0,
-        "end": 17
+        "start": 2,
+        "end": 18
       }
     },
     {
@@ -46,8 +46,8 @@
         "name": "key02",
         "span": {
           "type": "Span",
-          "start": 18,
-          "end": 23
+          "start": 20,
+          "end": 25
         }
       },
       "value": {
@@ -58,65 +58,100 @@
             "value": "Value 02\nContinued",
             "span": {
               "type": "Span",
-              "start": 31,
-              "end": 55
+              "start": 35,
+              "end": 58
             }
           }
         ],
         "span": {
           "type": "Span",
-          "start": 31,
-          "end": 55
+          "start": 35,
+          "end": 58
         }
       },
-      "attributes": [],
+      "attributes": [
+        {
+          "type": "Attribute",
+          "id": {
+            "type": "Identifier",
+            "name": "title",
+            "span": {
+              "type": "Span",
+              "start": 67,
+              "end": 72
+            }
+          },
+          "value": {
+            "type": "Pattern",
+            "elements": [
+              {
+                "type": "TextElement",
+                "value": "Title",
+                "span": {
+                  "type": "Span",
+                  "start": 75,
+                  "end": 80
+                }
+              }
+            ],
+            "span": {
+              "type": "Span",
+              "start": 75,
+              "end": 80
+            }
+          },
+          "span": {
+            "type": "Span",
+            "start": 66,
+            "end": 80
+          }
+        }
+      ],
       "comment": null,
       "span": {
         "type": "Span",
-        "start": 18,
-        "end": 55
+        "start": 20,
+        "end": 80
       }
     },
     {
       "type": "Comment",
-      "content": "ERROR Missing value or attributes",
+      "content": "ERROR Unclosed StringLiteral",
       "span": {
         "type": "Span",
-        "start": 58,
-        "end": 94
+        "start": 84,
+        "end": 114
       }
     },
     {
       "type": "Junk",
       "annotations": [
         {
           "type": "Annotation",
-          "code": "E0003",
-          "args": [
-            "="
-          ],
-          "message": "Expected token: \"=\"",
+          "code": "E0020",
+          "args": [],
+          "message": "Unterminated string expression",
           "span": {
             "type": "Span",
-            "start": 101,
-            "end": 101
+            "start": 130,
+            "end": 130
           }
         }
       ],
-      "content": "err03\r\n\r\n",
+      "content": "err03 = { \"str\r\n\r\n",
       "span": {
         "type": "Span",
-        "start": 95,
-        "end": 104
+        "start": 116,
+        "end": 134
       }
     },
     {
       "type": "Comment",
       "content": "ERROR Missing newline after ->.",
       "span": {
         "type": "Span",
-        "start": 104,
-        "end": 138
+        "start": 134,
+        "end": 167
       }
     },
     {
@@ -131,22 +166,22 @@
           "message": "Expected token: \"␤\"",
           "span": {
             "type": "Span",
-            "start": 157,
-            "end": 157
+            "start": 187,
+            "end": 187
           }
         }
       ],
       "content": "err04 = { $sel -> }\r\n",
       "span": {
         "type": "Span",
-        "start": 139,
-        "end": 160
+        "start": 169,
+        "end": 190
       }
     }
   ],
   "span": {
     "type": "Span",
     "start": 0,
-    "end": 160
+    "end": 190
   }
 }
diff --git a/fluent/test/fixtures_structure/crlf.json b/fluent/test/fixtures_structure/crlf.json
@@ -1,10 +1,15 @@
 {
     "key01": "Value 01",
-    "key02": [
-        "Value 02",
-        "\n",
-        "Continued"
-    ],
+    "key02": {
+        "value": [
+            "Value 02",
+            "\n",
+            "Continued"
+        ],
+        "attrs": {
+            "title": "Title"
+        }
+    },
     "err04": [
         {
             "type": "select",