End junk on blank lines to produce smaller Junk entries

projectfluent · Oct 19, 2018 · 946d5ac · 946d5ac
1 parent e5cf0c0
commit 946d5ac
Show file tree

Hide file tree

Showing 12 changed files with 296 additions and 44 deletions.
diff --git a/fluent-syntax/src/parser.js b/fluent-syntax/src/parser.js
@@ -135,18 +135,18 @@ export default class FluentParser {
       }
 
       let errorIndex = ps.index;
-      ps.skipToNextEntryStart(entryStartPos);
-      const nextEntryStart = ps.index;
-      if (nextEntryStart < errorIndex) {
+      ps.skipToJunkEnd(entryStartPos);
+      const junkEnd = ps.index;
+      if (junkEnd < errorIndex) {
         // The position of the error must be inside of the Junk's span.
-        errorIndex = nextEntryStart;
+        errorIndex = junkEnd;
       }
 
       // Create a Junk instance
-      const slice = ps.string.substring(entryStartPos, nextEntryStart);
+      const slice = ps.string.substring(entryStartPos, junkEnd);
       const junk = new AST.Junk(slice);
       if (this.withSpans) {
-        junk.addSpan(entryStartPos, nextEntryStart);
+        junk.addSpan(entryStartPos, junkEnd);
       }
       const annot = new AST.Annotation(err.code, err.args, err.message);
       annot.addSpan(errorIndex, errorIndex);

diff --git a/fluent-syntax/src/stream.js b/fluent-syntax/src/stream.js
@@ -87,6 +87,10 @@ export class FluentParserStream extends ParserStream {
       if (this.currentPeek === EOL) {
         this.next();
         lineCount++;
+      } else if (this.currentPeek === EOF) {
+        // Consume inline blanks before the EOF.
+        this.skipToPeek();
+        return lineCount;
       } else {
         this.resetPeek();
         return lineCount;
@@ -154,7 +158,7 @@ export class FluentParserStream extends ParserStream {
     return null;
   }
 
-  isCharIDStart(ch) {
+  isCharIdStart(ch) {
     if (ch === EOF) {
       return false;
     }
@@ -165,7 +169,7 @@ export class FluentParserStream extends ParserStream {
   }
 
   isIdentifierStart() {
-    return this.isCharIDStart(this.currentPeek);
+    return this.isCharIdStart(this.currentPeek);
   }
 
   isNumberStart() {
@@ -308,7 +312,7 @@ export class FluentParserStream extends ParserStream {
     return true;
   }
 
-  skipToNextEntryStart(junkStart) {
+  skipToJunkEnd(junkStart) {
     let lastNewline = this.string.lastIndexOf(EOL, this.index);
     if (junkStart < lastNewline) {
       // Last seen newline is _after_ the junk start. It's safe to rewind
@@ -322,16 +326,17 @@ export class FluentParserStream extends ParserStream {
         continue;
       }
 
-      // Break if the first char in this line looks like an entry start.
-      const first = this.next();
-      if (this.isCharIDStart(first) || first === "-" || first === "#") {
+      const ch = this.next();
+      // Break if the first char in this line looks like a beginning of a
+      // message, term or comment, or if it's a blank line.
+      if (this.isCharIdStart(ch) || ch === "-" || ch === "#" || ch === EOL) {
         break;
       }
     }
   }
 
   takeIDStart() {
-    if (this.isCharIDStart(this.currentChar)) {
+    if (this.isCharIdStart(this.currentChar)) {
       const ret = this.currentChar;
       this.next();
       return ret;

diff --git a/fluent-syntax/test/fixtures_structure/attribute_with_empty_pattern.json b/fluent-syntax/test/fixtures_structure/attribute_with_empty_pattern.json
@@ -16,11 +16,11 @@
           }
         }
       ],
-      "content": "key1 = Value 1\n    .attr =\n\n",
+      "content": "key1 = Value 1\n    .attr =\n",
       "span": {
         "type": "Span",
         "start": 0,
-        "end": 28
+        "end": 27
       }
     },
     {
@@ -38,11 +38,11 @@
           }
         }
       ],
-      "content": "key2 =\n    .attr =\n\n",
+      "content": "key2 =\n    .attr =\n",
       "span": {
         "type": "Span",
         "start": 28,
-        "end": 48
+        "end": 47
       }
     },
     {
@@ -60,11 +60,11 @@
           }
         }
       ],
-      "content": "key3 =\n    .attr1 = Attr 1\n    .attr2 =\n\n",
+      "content": "key3 =\n    .attr1 = Attr 1\n    .attr2 =\n",
       "span": {
         "type": "Span",
         "start": 48,
-        "end": 89
+        "end": 88
       }
     },
     {
@@ -82,11 +82,11 @@
           }
         }
       ],
-      "content": "key4 =\n    .attr1 =\n    .attr2 = Attr 2\n\n",
+      "content": "key4 =\n    .attr1 =\n    .attr2 = Attr 2\n",
       "span": {
         "type": "Span",
         "start": 89,
-        "end": 130
+        "end": 129
       }
     },
     {

diff --git a/fluent-syntax/test/fixtures_structure/junk.ftl b/fluent-syntax/test/fixtures_structure/junk.ftl
@@ -0,0 +1,18 @@
+err01 = {×××}
+err02 = {×××}
+
+err03 = {×××}
+×××
+
+err04 = {×××}
+
+×××
+
+err05 = {
+×××
+
+err06 = {
+
+key07 = Value
+
+err08 = {
diff --git a/fluent-syntax/test/fixtures_structure/junk.json b/fluent-syntax/test/fixtures_structure/junk.json
@@ -0,0 +1,226 @@
+{
+  "type": "Resource",
+  "body": [
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 9,
+            "end": 9
+          }
+        }
+      ],
+      "content": "err01 = {×××}\n",
+      "span": {
+        "type": "Span",
+        "start": 0,
+        "end": 14
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 23,
+            "end": 23
+          }
+        }
+      ],
+      "content": "err02 = {×××}\n",
+      "span": {
+        "type": "Span",
+        "start": 14,
+        "end": 28
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 38,
+            "end": 38
+          }
+        }
+      ],
+      "content": "err03 = {×××}\n×××\n",
+      "span": {
+        "type": "Span",
+        "start": 29,
+        "end": 47
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 57,
+            "end": 57
+          }
+        }
+      ],
+      "content": "err04 = {×××}\n",
+      "span": {
+        "type": "Span",
+        "start": 48,
+        "end": 62
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0002",
+          "args": [],
+          "message": "Expected an entry start",
+          "span": {
+            "type": "Span",
+            "start": 63,
+            "end": 63
+          }
+        }
+      ],
+      "content": "×××\n",
+      "span": {
+        "type": "Span",
+        "start": 63,
+        "end": 67
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 78,
+            "end": 78
+          }
+        }
+      ],
+      "content": "err05 = {\n×××\n",
+      "span": {
+        "type": "Span",
+        "start": 68,
+        "end": 82
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0003",
+          "args": [
+            "}"
+          ],
+          "message": "Expected token: \"}\"",
+          "span": {
+            "type": "Span",
+            "start": 94,
+            "end": 94
+          }
+        }
+      ],
+      "content": "err06 = {\n\n",
+      "span": {
+        "type": "Span",
+        "start": 83,
+        "end": 94
+      }
+    },
+    {
+      "type": "Message",
+      "id": {
+        "type": "Identifier",
+        "name": "key07",
+        "span": {
+          "type": "Span",
+          "start": 94,
+          "end": 99
+        }
+      },
+      "value": {
+        "type": "Pattern",
+        "elements": [
+          {
+            "type": "TextElement",
+            "value": "Value",
+            "span": {
+              "type": "Span",
+              "start": 102,
+              "end": 107
+            }
+          }
+        ],
+        "span": {
+          "type": "Span",
+          "start": 102,
+          "end": 107
+        }
+      },
+      "attributes": [],
+      "comment": null,
+      "span": {
+        "type": "Span",
+        "start": 94,
+        "end": 107
+      }
+    },
+    {
+      "type": "Junk",
+      "annotations": [
+        {
+          "type": "Annotation",
+          "code": "E0014",
+          "args": [],
+          "message": "Expected literal",
+          "span": {
+            "type": "Span",
+            "start": 119,
+            "end": 119
+          }
+        }
+      ],
+      "content": "err08 = {\n",
+      "span": {
+        "type": "Span",
+        "start": 109,
+        "end": 119
+      }
+    }
+  ],
+  "span": {
+    "type": "Span",
+    "start": 0,
+    "end": 119
+  }
+}