Merge pull request #46 from spahnke/regex-flags

Improve tokenization of regular expressions
microsoft · Oct 1, 2018 · 66b5497 · 66b5497
2 parents 0aea831 + ce3a065
commit 66b5497
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 3 deletions.
diff --git a/src/javascript/javascript.test.ts b/src/javascript/javascript.test.ts
@@ -430,7 +430,7 @@ testTokenization('javascript', [
 		]
 	}],
 
-[{
+	[{
 		line: 'x = /foo/.test(\'\')',
 		tokens: [
 			{ startIndex: 0, type: 'identifier.js' },
@@ -446,6 +446,39 @@ testTokenization('javascript', [
 		]
 	}],
 
+	[{
+		line: '/foo/',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/g',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' },
+			{ startIndex: 5, type: 'keyword.other.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/gimsuy',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.js' },
+			{ startIndex: 5, type: 'keyword.other.js' }
+		]
+	}],
+
+	[{
+		line: '/foo/q', // invalid flag
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.js' },
+			{ startIndex: 1, type: 'identifier.js' },
+			{ startIndex: 4, type: 'delimiter.js' },
+			{ startIndex: 5, type: 'identifier.js' }
+		]
+	}],
+
 	[{
 		line: 'x = 1 + f(2 / 3, /foo/)',
 		tokens: [

diff --git a/src/typescript/typescript.test.ts b/src/typescript/typescript.test.ts
@@ -446,6 +446,39 @@ testTokenization('typescript', [
 		]
 	}],
 
+	[{
+		line: '/foo/',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.ts' }
+		]
+	}],
+
+	[{
+		line: '/foo/g',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.ts' },
+			{ startIndex: 5, type: 'keyword.other.ts' }
+		]
+	}],
+
+	[{
+		line: '/foo/gimsuy',
+		tokens: [
+			{ startIndex: 0, type: 'regexp.ts' },
+			{ startIndex: 5, type: 'keyword.other.ts' }
+		]
+	}],
+
+	[{
+		line: '/foo/q', // invalid flag
+		tokens: [
+			{ startIndex: 0, type: 'delimiter.ts' },
+			{ startIndex: 1, type: 'identifier.ts' },
+			{ startIndex: 4, type: 'delimiter.ts' },
+			{ startIndex: 5, type: 'identifier.ts' }
+		]
+	}],
+
 	[{
 		line: 'x = 1 + f(2 / 3, /foo/)',
 		tokens: [

diff --git a/src/typescript/typescript.ts b/src/typescript/typescript.ts
@@ -130,7 +130,7 @@ export const language = {
 			{ include: '@whitespace' },
 
 			// regular expression: ensure it is terminated before beginning (otherwise it is an opeator)
-			[/\/(?=([^\\\/]|\\.)+\/([gimuy]*)(\s*)(\.|;|\/|,|\)|\]|\}|$))/, { token: 'regexp', bracket: '@open', next: '@regexp' }],
+			[/\/(?=([^\\\/]|\\.)+\/([gimsuy]*)(\s*)(\.|;|\/|,|\)|\]|\}|$))/, { token: 'regexp', bracket: '@open', next: '@regexp' }],
 
 			// delimiters and operators
 			[/[()\[\]]/, '@brackets'],
@@ -190,7 +190,7 @@ export const language = {
 			[/[^\\\/]/, 'regexp'],
 			[/@regexpesc/, 'regexp.escape'],
 			[/\\\./, 'regexp.invalid'],
-			['/', { token: 'regexp', bracket: '@close' }, '@pop'],
+			[/(\/)([gimsuy]*)/, [{ token: 'regexp', bracket: '@close', next: '@pop' }, 'keyword.other']],
 		],
 
 		regexrange: [