Add an easy way to run performance tests for tree-sitter

.vscode-test.js

@@ -42,6 +42,11 @@ const extensions = [
 		workspaceFolder: `extensions/vscode-colorize-tests/test`,
 		mocha: { timeout: 60_000 }
 	},
+	{
+		label: 'vscode-colorize-perf-tests',
+		workspaceFolder: `extensions/vscode-colorize-perf-tests/test`,
+		mocha: { timeout: 6000_000 }
+	},
 	{
 		label: 'configuration-editing',
 		workspaceFolder: path.join(os.tmpdir(), `confeditout-${Math.floor(Math.random() * 100000)}`),

.vscode/launch.json

@@ -202,6 +202,24 @@
 				"order": 5
 			}
 		},
+		{
+			"type": "extensionHost",
+			"request": "launch",
+			"name": "VS Code Tokenizer Performance Tests",
+			"runtimeExecutable": "${execPath}",
+			"args": [
+				"${workspaceFolder}/extensions/vscode-colorize-perf-tests/test",
+				"--extensionDevelopmentPath=${workspaceFolder}/extensions/vscode-colorize-perf-tests",
+				"--extensionTestsPath=${workspaceFolder}/extensions/vscode-colorize-perf-tests/out"
+			],
+			"outFiles": [
+				"${workspaceFolder}/out/**/*.js"
+			],
+			"presentation": {
+				"group": "5_tests",
+				"order": 6
+			}
+		},
 		{
 			"type": "chrome",
 			"request": "attach",

build/azure-pipelines/darwin/product-build-darwin-test.yml

@@ -61,6 +61,7 @@ steps:
           compile-extension:typescript-language-features \
           compile-extension:vscode-api-tests \
           compile-extension:vscode-colorize-tests \
+          compile-extension:vscode-colorize-perf-tests \
           compile-extension:vscode-test-resolver
       displayName: Build integration tests
 

build/azure-pipelines/linux/product-build-linux-test.yml

@@ -80,6 +80,7 @@ steps:
           compile-extension:typescript-language-features \
           compile-extension:vscode-api-tests \
           compile-extension:vscode-colorize-tests \
+          compile-extension:vscode-colorize-perf-tests \
           compile-extension:vscode-test-resolver
       displayName: Build integration tests
 

build/azure-pipelines/win32/product-build-win32-test.yml

@@ -63,6 +63,7 @@ steps:
           compile-extension:typescript-language-features `
           compile-extension:vscode-api-tests `
           compile-extension:vscode-colorize-tests `
+          compile-extension:vscode-colorize-perf-tests `
           compile-extension:vscode-test-resolver `
         }
       displayName: Build integration tests

build/gulpfile.extensions.js

@@ -65,6 +65,7 @@ const compilations = [
 	'extensions/typescript-language-features/tsconfig.json',
 	'extensions/vscode-api-tests/tsconfig.json',
 	'extensions/vscode-colorize-tests/tsconfig.json',
+	'extensions/vscode-colorize-perf-tests/tsconfig.json',
 	'extensions/vscode-test-resolver/tsconfig.json',
 
 	'.vscode/extensions/vscode-selfhost-test-provider/tsconfig.json',

build/lib/extensions.ts

@@ -278,6 +278,7 @@ export function fromGithub({ name, version, repo, sha256, metadata }: IExtension
 const excludedExtensions = [
 	'vscode-api-tests',
 	'vscode-colorize-tests',
+	'vscode-colorize-perf-tests',
 	'vscode-test-resolver',
 	'ms-vscode.node-debug',
 	'ms-vscode.node-debug2',

build/npm/dirs.js

@@ -44,6 +44,7 @@ const dirs = [
 	'extensions/typescript-language-features',
 	'extensions/vscode-api-tests',
 	'extensions/vscode-colorize-tests',
+	'extensions/vscode-colorize-perf-tests',
 	'extensions/vscode-test-resolver',
 	'remote',
 	'remote/web',

extensions/vscode-colorize-perf-tests/.gitignore

@@ -0,0 +1,2 @@
+out
+node_modules

extensions/vscode-colorize-perf-tests/.npmrc

@@ -0,0 +1,2 @@
+legacy-peer-deps="true"
+timeout=180000

extensions/vscode-colorize-perf-tests/.vscode/launch.json

@@ -0,0 +1,17 @@
+// A launch configuration that compiles the extension and then opens it inside a new window
+{
+	"version": "0.1.0",
+	"configurations": [
+		{
+			"name": "Launch Tests",
+			"type": "extensionHost",
+			"request": "launch",
+			"runtimeExecutable": "${execPath}",
+			"args": ["${workspaceFolder}/../../", "${workspaceFolder}/test", "--extensionDevelopmentPath=${workspaceFolder}", "--extensionTestsPath=${workspaceFolder}/out" ],
+			"stopOnEntry": false,
+			"sourceMaps": true,
+			"outDir": "${workspaceFolder}/out",
+			"preLaunchTask": "npm"
+		}
+	]
+}

extensions/vscode-colorize-perf-tests/.vscode/tasks.json

@@ -0,0 +1,11 @@
+{
+	"version": "2.0.0",
+	"command": "npm",
+	"type": "shell",
+	"presentation": {
+		"reveal": "silent"
+	},
+	"args": ["run", "compile"],
+	"isBackground": true,
+	"problemMatcher": "$tsc-watch"
+}

extensions/vscode-colorize-perf-tests/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "vscode-colorize-perf-tests",
+  "description": "Colorize performance tests for VS Code",
+  "version": "0.0.1",
+  "publisher": "vscode",
+  "license": "MIT",
+  "private": true,
+  "activationEvents": [
+    "onLanguage:json"
+  ],
+  "main": "./out/colorizerTestMain",
+  "engines": {
+    "vscode": "*"
+  },
+  "icon": "media/icon.png",
+  "scripts": {
+    "vscode:prepublish": "node ../../node_modules/gulp/bin/gulp.js --gulpfile ../../build/gulpfile.extensions.js compile-extension:vscode-colorize-perf-tests ./tsconfig.json",
+    "watch": "gulp watch-extension:vscode-colorize-perf-tests",
+    "compile": "gulp compile-extension:vscode-colorize-perf-tests"
+  },
+  "dependencies": {
+    "jsonc-parser": "^3.2.0"
+  },
+  "devDependencies": {
+    "@types/node": "20.x"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/vscode.git"
+  }
+}

extensions/vscode-colorize-perf-tests/src/colorizer.test.ts

@@ -0,0 +1,146 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as fs from 'fs';
+import 'mocha';
+import { basename, join, normalize } from 'path';
+import { commands, ConfigurationTarget, Uri, workspace } from 'vscode';
+
+interface BestsAndWorsts {
+	bestParse?: number;
+	bestCapture?: number;
+	bestMetadata?: number;
+	bestCombined: number;
+	worstParse?: number;
+	worstCapture?: number;
+	worstMetadata?: number;
+	worstCombined: number;
+}
+
+function findBestsAndWorsts(results: { parseTime?: number; captureTime?: number; metadataTime?: number; tokenizeTime?: number }[]): BestsAndWorsts {
+	let bestParse: number | undefined;
+	let bestCapture: number | undefined;
+	let bestMetadata: number | undefined;
+	let bestCombined: number | undefined;
+	let worstParse: number | undefined;
+	let worstCapture: number | undefined;
+	let worstMetadata: number | undefined;
+	let worstCombined: number | undefined;
+
+	for (let i = 0; i < results.length; i++) {
+		const result = results[i];
+		if (result.parseTime && result.captureTime && result.metadataTime) {
+			// Tree Sitter
+			const combined = result.parseTime + result.captureTime + result.metadataTime;
+			if (bestParse === undefined || result.parseTime < bestParse) {
+				bestParse = result.parseTime;
+			}
+			if (bestCapture === undefined || result.captureTime < bestCapture) {
+				bestCapture = result.captureTime;
+			}
+			if (bestMetadata === undefined || result.metadataTime < bestMetadata) {
+				bestMetadata = result.metadataTime;
+			}
+			if (bestCombined === undefined || combined < bestCombined) {
+				bestCombined = combined;
+			}
+			if (i !== 0) {
+				if (worstParse === undefined || result.parseTime > worstParse) {
+					worstParse = result.parseTime;
+				}
+				if (worstCapture === undefined || result.captureTime > worstCapture) {
+					worstCapture = result.captureTime;
+				}
+				if (worstMetadata === undefined || result.metadataTime > worstMetadata) {
+					worstMetadata = result.metadataTime;
+				}
+				if (worstCombined === undefined || combined > worstCombined) {
+					worstCombined = combined;
+				}
+			}
+		} else if (result.tokenizeTime) {
+			// TextMate
+			if (bestCombined === undefined || result.tokenizeTime < bestCombined) {
+				bestCombined = result.tokenizeTime;
+			}
+			if (i !== 0 && (worstCombined === undefined || result.tokenizeTime > worstCombined)) {
+				worstCombined = result.tokenizeTime;
+			}
+		}
+	}
+	return {
+		bestParse,
+		bestCapture,
+		bestMetadata,
+		bestCombined: bestCombined!,
+		worstParse,
+		worstCapture,
+		worstMetadata,
+		worstCombined: worstCombined!,
+	};
+}
+
+interface TreeSitterTimes {
+	parseTime: number;
+	captureTime: number;
+	metadataTime: number;
+}
+
+interface TextMateTimes {
+	tokenizeTime: number;
+}
+
+async function runCommand<TimesType = TreeSitterTimes | TextMateTimes>(command: string, file: Uri, times: number): Promise<TimesType[]> {
+	const results: TimesType[] = [];
+	for (let i = 0; i < times; i++) {
+		results.push(await commands.executeCommand(command, file));
+	}
+	return results;
+}
+
+async function doTest(file: Uri, times: number) {
+	const treeSitterResults = await runCommand<TreeSitterTimes>('_workbench.colorizeTreeSitterTokens', file, times);
+
+	const { bestParse, bestCapture, bestMetadata, bestCombined, worstParse, worstCapture, worstMetadata, worstCombined } = findBestsAndWorsts(treeSitterResults);
+	const textMateResults = await runCommand<TextMateTimes>('_workbench.colorizeTextMateTokens', file, times);
+	const textMateBestWorst = findBestsAndWorsts(textMateResults);
+
+	const toString = (time: number, charLength: number) => {
+		// truncate time to charLength characters
+		return time.toString().slice(0, charLength).padEnd(charLength, ' ');
+	};
+	const numLength = 7;
+	const resultString = `                        | First   | Best    | Worst   |
+| --------------------- | ------- | ------- | ------- |
+| TreeSitter (parse)    | ${toString(treeSitterResults[0].parseTime, numLength)} | ${toString(bestParse!, numLength)} | ${toString(worstParse!, numLength)} |
+| TreeSitter (capture)  | ${toString(treeSitterResults[0].captureTime, numLength)} | ${toString(bestCapture!, numLength)} | ${toString(worstCapture!, numLength)} |
+| TreeSitter (metadata) | ${toString(treeSitterResults[0].metadataTime, numLength)} | ${toString(bestMetadata!, numLength)} | ${toString(worstMetadata!, numLength)} |
+| TreeSitter (total)    | ${toString(treeSitterResults[0].parseTime + treeSitterResults[0].captureTime + treeSitterResults[0].metadataTime, numLength)} | ${toString(bestCombined, numLength)} | ${toString(worstCombined, numLength)} |
+| TextMate              | ${toString(textMateResults[0].tokenizeTime, numLength)} | ${toString(textMateBestWorst.bestCombined, numLength)} | ${toString(textMateBestWorst.worstCombined, numLength)} |
+`;
+	console.log(`File ${basename(file.fsPath)}:`);
+	console.log(resultString);
+}
+
+suite('Tokenization Performance', () => {
+	const testPath = normalize(join(__dirname, '../test'));
+	const fixturesPath = join(testPath, 'colorize-fixtures');
+	let originalSettingValue: any;
+
+	suiteSetup(async function () {
+		originalSettingValue = workspace.getConfiguration('editor').get('experimental.preferTreeSitter');
+		await workspace.getConfiguration('editor').update('experimental.preferTreeSitter', ["typescript"], ConfigurationTarget.Global);
+	});
+	suiteTeardown(async function () {
+		await workspace.getConfiguration('editor').update('experimental.preferTreeSitter', originalSettingValue, ConfigurationTarget.Global);
+	});
+
+	for (const fixture of fs.readdirSync(fixturesPath)) {
+		test(`Full file colorize: ${fixture}`, async function () {
+			await commands.executeCommand('workbench.action.closeAllEditors');
+			await doTest(Uri.file(join(fixturesPath, fixture)), 6);
+		});
+	}
+});

extensions/vscode-colorize-perf-tests/src/colorizerTestMain.ts

@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+
+export function activate(_context: vscode.ExtensionContext): any {
+
+}

extensions/vscode-colorize-perf-tests/src/index.ts

@@ -0,0 +1,30 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as path from 'path';
+import * as testRunner from '../../../test/integration/electron/testrunner';
+
+const suite = 'Performance Colorize Tests';
+
+const options: import('mocha').MochaOptions = {
+	ui: 'tdd',
+	color: true,
+	timeout: 60000
+};
+
+if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) {
+	options.reporter = 'mocha-multi-reporters';
+	options.reporterOptions = {
+		reporterEnabled: 'spec, mocha-junit-reporter',
+		mochaJunitReporterReporterOptions: {
+			testsuitesTitle: `${suite} ${process.platform}`,
+			mochaFile: path.join(process.env.BUILD_ARTIFACTSTAGINGDIRECTORY, `test-results/${process.platform}-${process.arch}-${suite.toLowerCase().replace(/[^\w]/g, '-')}-results.xml`)
+		}
+	};
+}
+
+testRunner.configure(options);
+
+export = testRunner;