Remove escape sequences from raw strings (Fixes #67) (#69)

nishtahir · May 19, 2023 · bad0234 · bad0234
1 parent f47a0c4
commit bad0234
Show file tree

Hide file tree

Showing 13 changed files with 186 additions and 56 deletions.
diff --git a/dist/Kotlin.JSON-tmLanguage b/dist/Kotlin.JSON-tmLanguage
@@ -795,7 +795,7 @@
               "name": "string.quoted.triple.kotlin",
               "patterns": [
                 {
-                  "include": "#string-content"
+                  "include": "#raw-string-content"
                 }
               ]
             },
@@ -841,6 +841,34 @@
             }
           ],
           "repository": {
+            "raw-string-content": {
+              "patterns": [
+                {
+                  "begin": "\\$(\\{)",
+                  "beginCaptures": {
+                    "1": {
+                      "name": "punctuation.section.block.begin.kotlin"
+                    }
+                  },
+                  "end": "\\}",
+                  "endCaptures": {
+                    "0": {
+                      "name": "punctuation.section.block.end.kotlin"
+                    }
+                  },
+                  "name": "entity.string.template.element.kotlin",
+                  "patterns": [
+                    {
+                      "include": "#code"
+                    }
+                  ]
+                },
+                {
+                  "match": "\\$[a-zA-Z_]\\w*",
+                  "name": "entity.string.template.element.kotlin"
+                }
+              ]
+            },
             "string-content": {
               "patterns": [
                 {

diff --git a/dist/Kotlin.YAML-tmLanguage b/dist/Kotlin.YAML-tmLanguage
@@ -324,10 +324,11 @@ repository:
                     - {match: '\b([0-9][0-9_]*([fFLuU]|[uU]L)?)\b', name: constant.numeric.integer.kotlin}
             string:
                 patterns:
-                    - {begin: '"""', beginCaptures: {'0': {name: punctuation.definition.string.begin.kotlin}}, end: '"""(?!")', endCaptures: {'0': {name: punctuation.definition.string.end.kotlin}}, name: string.quoted.triple.kotlin, patterns: [{include: '#string-content'}]}
+                    - {begin: '"""', beginCaptures: {'0': {name: punctuation.definition.string.begin.kotlin}}, end: '"""(?!")', endCaptures: {'0': {name: punctuation.definition.string.end.kotlin}}, name: string.quoted.triple.kotlin, patterns: [{include: '#raw-string-content'}]}
                     - {begin: '(?!'')"', beginCaptures: {'0': {name: punctuation.definition.string.begin.kotlin}}, end: '"', endCaptures: {'0': {name: punctuation.definition.string.end.kotlin}}, name: string.quoted.double.kotlin, patterns: [{include: '#string-content'}]}
                     - {begin: '''', beginCaptures: {'0': {name: punctuation.definition.string.begin.kotlin}}, end: '''', endCaptures: {'0': {name: punctuation.definition.string.end.kotlin}}, name: string.quoted.single.kotlin, patterns: [{include: '#string-content'}]}
                 repository:
+                    raw-string-content: {patterns: [{begin: '\$(\{)', beginCaptures: {'1': {name: punctuation.section.block.begin.kotlin}}, end: '\}', endCaptures: {'0': {name: punctuation.section.block.end.kotlin}}, name: entity.string.template.element.kotlin, patterns: [{include: '#code'}]}, {match: '\$[a-zA-Z_]\w*', name: entity.string.template.element.kotlin}]}
                     string-content: {patterns: [{match: '\\[0\\tnr"'']', name: constant.character.escape.kotlin}, {match: '\\(x[\da-fA-F]{2}|u[\da-fA-F]{4}|.)', name: constant.character.escape.unicode.kotlin}, {begin: '\$(\{)', beginCaptures: {'1': {name: punctuation.section.block.begin.kotlin}}, end: '\}', endCaptures: {'0': {name: punctuation.section.block.end.kotlin}}, name: entity.string.template.element.kotlin, patterns: [{include: '#code'}]}, {match: '\$[a-zA-Z_]\w*', name: entity.string.template.element.kotlin}]}
             'null':
                 patterns:

diff --git a/dist/Kotlin.tmLanguage b/dist/Kotlin.tmLanguage
@@ -1200,7 +1200,7 @@
                 <array>
                   <dict>
                     <key>include</key>
-                    <string>#string-content</string>
+                    <string>#raw-string-content</string>
                   </dict>
                 </array>
               </dict>
@@ -1269,6 +1269,49 @@
             </array>
             <key>repository</key>
             <dict>
+              <key>raw-string-content</key>
+              <dict>
+                <key>patterns</key>
+                <array>
+                  <dict>
+                    <key>begin</key>
+                    <string>\$(\{)</string>
+                    <key>beginCaptures</key>
+                    <dict>
+                      <key>1</key>
+                      <dict>
+                        <key>name</key>
+                        <string>punctuation.section.block.begin.kotlin</string>
+                      </dict>
+                    </dict>
+                    <key>end</key>
+                    <string>\}</string>
+                    <key>endCaptures</key>
+                    <dict>
+                      <key>0</key>
+                      <dict>
+                        <key>name</key>
+                        <string>punctuation.section.block.end.kotlin</string>
+                      </dict>
+                    </dict>
+                    <key>name</key>
+                    <string>entity.string.template.element.kotlin</string>
+                    <key>patterns</key>
+                    <array>
+                      <dict>
+                        <key>include</key>
+                        <string>#code</string>
+                      </dict>
+                    </array>
+                  </dict>
+                  <dict>
+                    <key>match</key>
+                    <string>\$[a-zA-Z_]\w*</string>
+                    <key>name</key>
+                    <string>entity.string.template.element.kotlin</string>
+                  </dict>
+                </array>
+              </dict>
               <key>string-content</key>
               <dict>
                 <key>patterns</key>

diff --git a/package.json b/package.json
@@ -13,13 +13,14 @@
     "url": "https://github.com/nishtahir/language-kotlin/issues"
   },
   "scripts": {
-    "build": "node scripts/build.js",
+    "build:snapshot": "npx vscode-tmgrammar-snap -u -g dist/Kotlin.tmLanguage -s source.kotlin \"snapshots/**/*.kt\"",
+    "build:grammar": "node scripts/build.js",
+    "build": "npm run build:grammar && npm run build:snapshot",
     "format": "node scripts/format.js",
     "coverage": "node scripts/coverage.js && npx codecov",
-    "sample": "npx vscode-tmgrammar-snap --help",
-    "snapshot": "npx vscode-tmgrammar-snap -g dist/Kotlin.tmLanguage -s source.kotlin \"snapshots/**/*.kt\"",
-    "grammar-test": "npx vscode-tmgrammar-test -g dist/Kotlin.tmLanguage \"test/**/*.test.kt\"",
-    "test": "npm run grammar-test && npm run snapshot"
+    "test:snapshot": "npx vscode-tmgrammar-snap -g dist/Kotlin.tmLanguage -s source.kotlin \"snapshots/**/*.kt\"",
+    "test:grammar": "npx vscode-tmgrammar-test -g dist/Kotlin.tmLanguage \"test/**/*.test.kt\"",
+    "test": "npm run test:grammar && npm run test:snapshot"
   },
   "devDependencies": {
     "codecov": "^3.8.3",

diff --git a/scripts/build.js b/scripts/build.js
@@ -1,3 +1,11 @@
+/**
+ * This script is used to generate the final syntax files from the source files.
+ * It merges all source files into a single file and generates the final syntax files.
+ * - Kotlin.JSON-tmLanguage
+ * - Kotlin.YAML-tmLanguage
+ * - Kotlin.tmLanguage
+ */
+
 const fs = require("fs");
 const path = require("path");
 const plist = require("plist");

diff --git a/scripts/coverage.js b/scripts/coverage.js
@@ -1,3 +1,14 @@
+/**
+ * This script generates a naive coverage report for the Kotlin.tmlanguage file.
+ * It is used to ensure that all scopes are tested.
+ * 
+ * The script works by instrumenting the compiled Kotlin.tmlanguage file with
+ * coverage tracking scopes. It then runs all tests against the instrumented file
+ * and generates a coverage report.
+ * 
+ * It does not consider the inner structure of the scopes, only that they are hit.
+ */
+
 const path = require("path");
 const plist = require("plist");
 const {

diff --git a/scripts/format.js b/scripts/format.js
@@ -1,3 +1,8 @@
+/**
+ * This script formats all YAML files in the src/ directory.
+ * It is used to ensure a consistent code style.
+ */
+
 const fs = require("fs");
 const path = require("path");
 const yaml = require("yamljs");

diff --git a/src/classes.YAML-tmLanguage b/src/classes.YAML-tmLanguage
@@ -9,7 +9,7 @@ repository:
                     -
                         include: '#keywords'
                     -
-                        begin: \b((?:(?:data|value)\s+)?class|(?:(?:fun|value)\s+)?interface)\b\s+(\w+)
+                        begin: '\b((?:(?:data|value)\s+)?class|(?:(?:fun|value)\s+)?interface)\b\s+(\w+)'
                         beginCaptures:
                             '1':
                                 name: storage.modifier.kotlin
@@ -76,5 +76,5 @@ repository:
                         name: punctuation.seperator.kotlin
                     -
                         include: '#types'
-                    -   
+                    -
                         include: '#literals'
diff --git a/src/generic.YAML-tmLanguage b/src/generic.YAML-tmLanguage
@@ -1,38 +1,37 @@
-repository:
-    generic:
-        patterns:
-            -
-                begin: '(?=\<(?:[A-Z_]|\*|in|out))'
-                end: '(?<=\>)(?!\>)'
-                patterns:
-                    -
-                        match: '<'
-                        name: punctuation.bracket.angle.begin.kotlin
-                    -
-                        match: '>'
-                        name: punctuation.bracket.angle.end.kotlin
-                    -
-                        match: '\*'
-                        name: entity.name.type.generic.wildcard.kotlin
-                    -
-                        include: '#generic-parameter-list'
-                    -
-                        match: ','
-                        name: punctuation.seperator.kotlin
-
-        repository:
-            generic-parameter-list:
-                patterns:
-                    -
-                        include: '#annotations'
-                    -
-                        match: '\b(in|out)\b'
-                        name: storage.modifier.generic.variance.kotlin
-                    -
-                        include: '#built-in-types'
-                    -
-                        include: '#class-ident'
-                    -
-                        include: '#generic'
-                    -
-                        include: '#operators'
+repository:
+    generic:
+        patterns:
+            -
+                begin: '(?=\<(?:[A-Z_]|\*|in|out))'
+                end: '(?<=\>)(?!\>)'
+                patterns:
+                    -
+                        match: '<'
+                        name: punctuation.bracket.angle.begin.kotlin
+                    -
+                        match: '>'
+                        name: punctuation.bracket.angle.end.kotlin
+                    -
+                        match: '\*'
+                        name: entity.name.type.generic.wildcard.kotlin
+                    -
+                        include: '#generic-parameter-list'
+                    -
+                        match: ','
+                        name: punctuation.seperator.kotlin
+        repository:
+            generic-parameter-list:
+                patterns:
+                    -
+                        include: '#annotations'
+                    -
+                        match: \b(in|out)\b
+                        name: storage.modifier.generic.variance.kotlin
+                    -
+                        include: '#built-in-types'
+                    -
+                        include: '#class-ident'
+                    -
+                        include: '#generic'
+                    -
+                        include: '#operators'
diff --git a/src/ident.YAML-tmLanguage b/src/ident.YAML-tmLanguage
@@ -1,5 +1,3 @@
-# remove match to original master branch
-
 repository:
     class-ident:
         patterns:

diff --git a/src/keywords.YAML-tmLanguage b/src/keywords.YAML-tmLanguage
@@ -14,21 +14,21 @@ repository:
                 match: \b(\_)\b
                 name: punctuation.definition.variable.kotlin
             -
-                match: \b(tailrec|operator|infix|typealias|reified|copy(?=\s+fun|\s+var))\b
+                match: '\b(tailrec|operator|infix|typealias|reified|copy(?=\s+fun|\s+var))\b'
                 name: storage.type.kotlin
             -
                 match: \b(out|in|yield|typealias|override)\b
                 name: storage.modifier.kotlin
             -
-                match: \b(?<![+-/%*=(,]\s)(inline|inner|external|public|private|protected|internal|abstract|final|sealed|enum|open|annotation|expect|actual|const|lateinit)(?=\s(?!(?:\s*)(?:[+-/%*=:).,]|$)))\b
+                match: '\b(?<![+-/%*=(,]\s)(inline|inner|external|public|private|protected|internal|abstract|final|sealed|enum|open|annotation|expect|actual|const|lateinit)(?=\s(?!(?:\s*)(?:[+-/%*=:).,]|$)))\b'
                 name: storage.modifier.kotlin
                 comment: 'Soft modifiers'
             -
-                match: \b(vararg(?=\s+\w+:))\b
+                match: '\b(vararg(?=\s+\w+:))\b'
                 name: storage.modifier.kotlin
                 comment: 'Soft modifier'
             -
-                match: \b(suspend(?!\s*[\(]?\s*\{))\b
+                match: '\b(suspend(?!\s*[\(]?\s*\{))\b'
                 name: storage.modifier.kotlin
                 comment: 'Soft modifier'
             -
@@ -46,7 +46,6 @@ repository:
             -
                 match: \b(companion|object)\b
                 name: storage.type.kotlin
-
     operators:
         patterns:
             -

diff --git a/src/literals.YAML-tmLanguage b/src/literals.YAML-tmLanguage
@@ -43,7 +43,7 @@ repository:
                         name: string.quoted.triple.kotlin
                         patterns:
                             -
-                                include: '#string-content'
+                                include: '#raw-string-content'
                     -
                         begin: '(?!'')"'
                         beginCaptures:
@@ -71,6 +71,26 @@ repository:
                             -
                                 include: '#string-content'
                 repository:
+                    # https://kotlinlang.org/docs/strings.html#raw-strings
+                    # Raw strings do not contain escape sequences.
+                    raw-string-content:
+                        patterns:
+                            -
+                                begin: '\$(\{)'
+                                beginCaptures:
+                                    '1':
+                                        name: punctuation.section.block.begin.kotlin
+                                end: '\}'
+                                endCaptures:
+                                    '0':
+                                        name: punctuation.section.block.end.kotlin
+                                name: entity.string.template.element.kotlin
+                                patterns:
+                                    -
+                                        include: '#code'
+                            -
+                                match: '\$[a-zA-Z_]\w*'
+                                name: entity.string.template.element.kotlin
                     string-content:
                         patterns:
                             -

diff --git a/test/regression/67.test.kt b/test/regression/67.test.kt
@@ -0,0 +1,17 @@
+// SYNTAX TEST "source.kotlin" "Triple-quoted strings do not ignore backslash before closing sequence"
+
+>val cmd = """bash -c "sudo -u test bash --noprofile --norc -euo pipefail "\"""
+#^^^ source.kotlin storage.type.kotlin
+#   ^^^^^ source.kotlin
+#        ^ source.kotlin keyword.operator.assignment.kotlin
+#         ^ source.kotlin
+#          ^^^ source.kotlin string.quoted.triple.kotlin punctuation.definition.string.begin.kotlin
+#             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ source.kotlin string.quoted.triple.kotlin
+#                                                                           ^^^ source.kotlin string.quoted.triple.kotlin punctuation.definition.string.end.kotlin
+>val num = 1
+#^^^ source.kotlin storage.type.kotlin
+#   ^^^^^ source.kotlin
+#        ^ source.kotlin keyword.operator.assignment.kotlin
+#         ^ source.kotlin
+#          ^ source.kotlin constant.numeric.integer.kotlin
+>