From 93c3d0e224aeead1abf98261d3d318273095233a Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 12:26:49 +0200 Subject: [PATCH 01/47] wip --- .../javascript-language-configuration.json | 4 +- .../language-configuration.json | 4 +- src/vs/editor/common/languages/autoIndent.ts | 69 +++--- src/vs/editor/common/languages/enterAction.ts | 27 +-- .../common/languages/languageConfiguration.ts | 2 +- .../languages/lineProcessorForIndentation.ts | 215 ++++++++++++++++++ src/vs/editor/common/languages/supports.ts | 20 +- 7 files changed, 269 insertions(+), 72 deletions(-) create mode 100644 src/vs/editor/common/languages/lineProcessorForIndentation.ts diff --git a/extensions/javascript/javascript-language-configuration.json b/extensions/javascript/javascript-language-configuration.json index fb2fb0397d790..f7c332337cb8e 100644 --- a/extensions/javascript/javascript-language-configuration.json +++ b/extensions/javascript/javascript-language-configuration.json @@ -111,10 +111,10 @@ }, "indentationRules": { "decreaseIndentPattern": { - "pattern": "^((?!.*?/\\*).*\\*\/)?\\s*[\\}\\]\\)].*$" + "pattern": "^\\s*[\\}\\]\\)].*$" }, "increaseIndentPattern": { - "pattern": "^((?!//).)*(\\{([^}\"'`/]*|(\\t|[ ])*//.*)|\\([^)\"'`/]*|\\[[^\\]\"'`/]*)$" + "pattern": "^.*(\\{[^}]*|\\([^)]*|\\[[^\\]]*)$" }, // e.g. * ...| or */| or *-----*/| "unIndentedLinePattern": { diff --git a/extensions/typescript-basics/language-configuration.json b/extensions/typescript-basics/language-configuration.json index 070b8911a825e..25a2368573841 100644 --- a/extensions/typescript-basics/language-configuration.json +++ b/extensions/typescript-basics/language-configuration.json @@ -129,10 +129,10 @@ }, "indentationRules": { "decreaseIndentPattern": { - "pattern": "^((?!.*?/\\*).*\\*\/)?\\s*[\\}\\]\\)].*$" + "pattern": "^\\s*[\\}\\]\\)].*$" }, "increaseIndentPattern": { - "pattern": "^((?!//).)*(\\{([^}\"'`/]*|(\\t|[ ])*//.*)|\\([^)\"'`/]*|\\[[^\\]\"'`/]*)$" + "pattern": "^.*(\\{[^}]*|\\([^)]*|\\[[^\\]]*)$" }, // e.g. * ...| or */| or *-----*/| "unIndentedLinePattern": { diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 9ae3df974aa59..a73c341535223 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -12,6 +12,7 @@ import { IndentConsts, IndentRulesSupport } from 'vs/editor/common/languages/sup import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { ScopedLineProcessorForIndentation, LineProcessorForIndentation, withinEmbeddedLanguage } from 'vs/editor/common/languages/lineProcessorForIndentation'; export interface IVirtualModel { tokenization: { @@ -116,7 +117,8 @@ export function getInheritIndentForLine( }; } - const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); + const linesProcessor = new LineProcessorForIndentation(model, languageConfigurationService); + const precedingUnIgnoredLineContent = linesProcessor.getLine(precedingUnIgnoredLine); if (indentRulesSupport.shouldIncrease(precedingUnIgnoredLineContent) || indentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLineContent)) { return { indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent), @@ -150,7 +152,7 @@ export function getInheritIndentForLine( (previousLineIndentMetadata & IndentConsts.INDENT_NEXTLINE_MASK)) { let stopLine = 0; for (let i = previousLine - 1; i > 0; i--) { - if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) { + if (indentRulesSupport.shouldIndentNextLine(linesProcessor.getLine(i))) { continue; } stopLine = i; @@ -173,7 +175,7 @@ export function getInheritIndentForLine( } else { // search from precedingUnIgnoredLine until we find one whose indent is not temporary for (let i = precedingUnIgnoredLine; i > 0; i--) { - const lineContent = model.getLineContent(i); + const lineContent = linesProcessor.getLine(i); if (indentRulesSupport.shouldIncrease(lineContent)) { return { indentation: strings.getLeadingWhitespace(lineContent), @@ -183,7 +185,7 @@ export function getInheritIndentForLine( } else if (indentRulesSupport.shouldIndentNextLine(lineContent)) { let stopLine = 0; for (let j = i - 1; j > 0; j--) { - if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) { + if (indentRulesSupport.shouldIndentNextLine(linesProcessor.getLine(i))) { continue; } stopLine = j; @@ -191,7 +193,7 @@ export function getInheritIndentForLine( } return { - indentation: strings.getLeadingWhitespace(model.getLineContent(stopLine + 1)), + indentation: strings.getLeadingWhitespace(linesProcessor.getLine(stopLine + 1)), action: null, line: stopLine + 1 }; @@ -205,7 +207,7 @@ export function getInheritIndentForLine( } return { - indentation: strings.getLeadingWhitespace(model.getLineContent(1)), + indentation: strings.getLeadingWhitespace(linesProcessor.getLine(1)), action: null, line: 1 }; @@ -236,7 +238,8 @@ export function getGoodIndentForLine( } const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); - const lineContent = virtualModel.getLineContent(lineNumber); + const linesProcessor = new LineProcessorForIndentation(virtualModel, languageConfigurationService); + const lineContent = linesProcessor.getLine(lineNumber); if (indent) { const inheritLine = indent.line; @@ -244,16 +247,16 @@ export function getGoodIndentForLine( // Apply enter action as long as there are only whitespace lines between inherited line and this line. let shouldApplyEnterRules = true; for (let inBetweenLine = inheritLine; inBetweenLine < lineNumber - 1; inBetweenLine++) { - if (!/^\s*$/.test(virtualModel.getLineContent(inBetweenLine))) { + if (!/^\s*$/.test(linesProcessor.getLine(inBetweenLine))) { shouldApplyEnterRules = false; break; } } if (shouldApplyEnterRules) { - const enterResult = richEditSupport.onEnter(autoIndent, '', virtualModel.getLineContent(inheritLine), ''); + const enterResult = richEditSupport.onEnter(autoIndent, '', linesProcessor.getLine(inheritLine), ''); if (enterResult) { - let indentation = strings.getLeadingWhitespace(virtualModel.getLineContent(inheritLine)); + let indentation = strings.getLeadingWhitespace(linesProcessor.getLine(inheritLine)); if (enterResult.removeText) { indentation = indentation.substring(0, indentation.length - enterResult.removeText); @@ -311,25 +314,16 @@ export function getIndentForEnter( model.tokenization.forceTokenization(range.startLineNumber); const lineTokens = model.tokenization.getLineTokens(range.startLineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const scopedLineText = scopedLineTokens.getLineContent(); - - let embeddedLanguage = false; - let beforeEnterText: string; - if (scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId) { - // we are in the embeded language content - embeddedLanguage = true; // if embeddedLanguage is true, then we don't touch the indentation of current line - beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - beforeEnterText = lineTokens.getLineContent().substring(0, range.startColumn - 1); - } + const processLines = new ScopedLineProcessorForIndentation(model, languageConfigurationService); + const embeddedLanguage = withinEmbeddedLanguage(lineTokens, scopedLineTokens); - let afterEnterText: string; - if (range.isEmpty()) { - afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } + const processedBeforeLines = processLines.getProcessedLineBeforeRange(range, scopedLineTokens); + const beforeEnterText = processedBeforeLines.processedLine; + const processedAfterLines = processLines.getProcessedLineAfterRange(range, scopedLineTokens); + const afterEnterText = processedAfterLines.processedLine; + + console.log('beforeEnterText : ', beforeEnterText); + console.log('afterEnterText : ', afterEnterText); const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId).indentRulesSupport; if (!indentRulesSupport) { @@ -413,21 +407,18 @@ export function getIndentActionForType( return null; } - const scopedLineText = scopedLineTokens.getLineContent(); - const beforeTypeText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); + const linesProcessor = new ScopedLineProcessorForIndentation(model, languageConfigurationService); + const beforeEnterResult = linesProcessor.getProcessedLineBeforeRange(range, scopedLineTokens); + const afterEnterResult = linesProcessor.getProcessedLineAfterRange(range, scopedLineTokens); + const fullText = beforeEnterResult.processedLine + afterEnterResult.processedLine; + const fullTextWithCharacter = beforeEnterResult.processedLine + ch + afterEnterResult.processedLine; - // selection support - let afterTypeText: string; - if (range.isEmpty()) { - afterTypeText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterTypeText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } + console.log('fullText : ', fullText); + console.log('fullTextWithCharacter : ', fullTextWithCharacter); // If previous content already matches decreaseIndentPattern, it means indentation of this line should already be adjusted // Users might change the indentation by purpose and we should honor that instead of readjusting. - if (!indentRulesSupport.shouldDecrease(beforeTypeText + afterTypeText) && indentRulesSupport.shouldDecrease(beforeTypeText + ch + afterTypeText)) { + if (!indentRulesSupport.shouldDecrease(fullText) && indentRulesSupport.shouldDecrease(fullTextWithCharacter)) { // after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner. // 1. Get inherited indent action const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageConfigurationService); diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index 447665fe81687..64ce04bed2112 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -8,6 +8,7 @@ import { ITextModel } from 'vs/editor/common/model'; import { IndentAction, CompleteEnterAction } from 'vs/editor/common/languages/languageConfiguration'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getIndentationAtPosition, getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ScopedLineProcessorForIndentation } from 'vs/editor/common/languages/lineProcessorForIndentation'; export function getEnterAction( autoIndent: EditorAutoIndentStrategy, @@ -20,28 +21,10 @@ export function getEnterAction( if (!richEditSupport) { return null; } - - const scopedLineText = scopedLineTokens.getLineContent(); - const beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); - - // selection support - let afterEnterText: string; - if (range.isEmpty()) { - afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } - - let previousLineText = ''; - if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { - // This is not the first line and the entire line belongs to this mode - const oneLineAboveScopedLineTokens = getScopedLineTokens(model, range.startLineNumber - 1); - if (oneLineAboveScopedLineTokens.languageId === scopedLineTokens.languageId) { - // The line above ends with text belonging to the same mode - previousLineText = oneLineAboveScopedLineTokens.getLineContent(); - } - } + const processLines = new ScopedLineProcessorForIndentation(model, languageConfigurationService); + const beforeEnterText = processLines.getProcessedLineBeforeRange(range, scopedLineTokens).processedLine; + const afterEnterText = processLines.getProcessedLineAfterRange(range, scopedLineTokens).processedLine; + const previousLineText = processLines.getProcessedPreviousLine(range); const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText); if (!enterResult) { diff --git a/src/vs/editor/common/languages/languageConfiguration.ts b/src/vs/editor/common/languages/languageConfiguration.ts index 69b852e407c4f..7ecf1e1f71c97 100644 --- a/src/vs/editor/common/languages/languageConfiguration.ts +++ b/src/vs/editor/common/languages/languageConfiguration.ts @@ -321,7 +321,7 @@ export class StandardAutoClosingPairConditional { public shouldAutoClose(context: ScopedLineTokens, column: number): boolean { // Always complete on empty line - if (context.getTokenCount() === 0) { + if (context.getCount() === 0) { return true; } diff --git a/src/vs/editor/common/languages/lineProcessorForIndentation.ts b/src/vs/editor/common/languages/lineProcessorForIndentation.ts new file mode 100644 index 0000000000000..72776228e8652 --- /dev/null +++ b/src/vs/editor/common/languages/lineProcessorForIndentation.ts @@ -0,0 +1,215 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Range } from 'vs/editor/common/core/range'; +import { ITextModel } from 'vs/editor/common/model'; +import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ScopedLineTokens } from 'vs/editor/common/languages/supports'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; +import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; + +type LineTokensType = LineTokens | ScopedLineTokens; + +interface ProcessedLineForIndentation { + line: string; + processedLine: string; +} + +export class LineProcessorForIndentation { + + constructor( + protected readonly model: IVirtualModel, + protected readonly languageConfigurationService: ILanguageConfigurationService + ) { } + + getLine(lineNumber: number): string { + + const languageId = this.model.tokenization.getLanguageId(); + const lineContent = this.model.getLineContent(lineNumber); + const tokens = this.model.tokenization.getLineTokens(lineNumber); + const processedLine = this.getProcessedLineForLineAndTokens(languageId, lineContent, tokens); + + console.log('getStrippedLine'); + console.log('lineContent : ', lineContent); + console.log('processedLine : ', processedLine); + + return processedLine; + } + + protected getProcessedLineForLineAndTokens(languageId: string, line: string, tokens: LineTokensType): string { + + const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, line: string): { processedCharacterOffset: number, processedLine: string } => { + const result = removeBracketsFromTokenWithIndex(tokenIndex); + const processedCharacterOffset = characterOffset + result.tokenText.length - result.processedText.length; + const processedLine = line.substring(0, characterOffset + result.tokenStartCharacterOffset) + result.processedText + line.substring(characterOffset + result.tokenEndCharacterOffset); + return { processedCharacterOffset, processedLine }; + }; + const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { + const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); + const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); + const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); + const processedText = removeBracketsFromText(tokenText); + return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; + } + const removeBracketsFromText = (line: string): string => { + let processedLine = line; + openBrackets.forEach((bracket) => { + processedLine = processedLine.replace(bracket, '_'); + }); + closedBrackets.forEach((bracket) => { + processedLine = processedLine.replace(bracket, '_'); + }); + return processedLine; + } + + console.log('getStrippedLineForLineAndTokens'); + console.log('line : ', line); + console.log('languageID : ', languageId); + + const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; + if (!brackets) { + return line; + } + + const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); + const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); + + let characterOffset = 0; + let processedLine = line; + + for (let i = 0; i < tokens.getCount(); i++) { + + console.log('i : ', i); + + const standardTokenType = tokens.getStandardTokenType(i); + if (standardTokenType === StandardTokenType.String + || standardTokenType === StandardTokenType.RegEx + || standardTokenType === StandardTokenType.Comment + ) { + const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); + characterOffset = result.processedCharacterOffset; + processedLine = result.processedLine; + } + } + return processedLine; + } +} + +export class ScopedLineProcessorForIndentation extends LineProcessorForIndentation { + + protected override readonly model: ITextModel; + + constructor( + model: ITextModel, + languageConfigurationService: ILanguageConfigurationService + ) { + super(model, languageConfigurationService); + this.model = model; + } + + getProcessedLineAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineForIndentation { + let columnIndexWithinScope: number; + let lineTokens: LineTokens; + // let afterScopedLineTokens: ScopedLineTokens + if (range.isEmpty()) { + columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; + lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + // afterScopedLineTokens = scopedLineTokens; + } else { + columnIndexWithinScope = range.endColumn - 1 - scopedLineTokens.firstCharOffset; + lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); + // afterScopedLineTokens = getScopedLineTokens(this.model, range.endLineNumber, range.endColumn); + } + return this.getProcessedScopedLine(lineTokens, scopedLineTokens, { isStart: true, columnIndexWithinScope }); + } + + getProcessedLineBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineForIndentation { + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + let columnIndexWithinScope: number; + let scopedLineTokensOfInterest: ScopedLineTokens | LineTokens; + if (withinEmbeddedLanguage(lineTokens, scopedLineTokens)) { + // we are in the embeded language content + columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; + scopedLineTokensOfInterest = scopedLineTokens; + } else { + columnIndexWithinScope = range.startColumn - 1; + scopedLineTokensOfInterest = lineTokens; + } + return this.getProcessedScopedLine(lineTokens, scopedLineTokensOfInterest, { isStart: false, columnIndexWithinScope }); + } + + getProcessedPreviousLine(range: Range) { + let processedPreviousLine = ''; + const scopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber, range.startColumn); + const language = this.model.tokenization.getLanguageId(); + if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { + // This is not the first line and the entire line belongs to this mode + const previousLineScopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber - 1); + if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { + // The line above ends with text belonging to the same mode + const previousLine = previousLineScopedLineTokens.getLineContent(); + processedPreviousLine = this.getProcessedLineForLineAndTokens(language, previousLine, previousLineScopedLineTokens); + } + } + console.log('previousLineText : ', processedPreviousLine); + return processedPreviousLine; + } + + private getProcessedScopedLine(initialLineTokens: LineTokens, scopedLineTokens: LineTokensType, opts: { columnIndexWithinScope: number, isStart: boolean }): ProcessedLineForIndentation { + + let languageId: string; + if (scopedLineTokens instanceof LineTokens) { + languageId = scopedLineTokens.getLanguageId(0); + } else { + languageId = scopedLineTokens.languageId; + } + + const scopedLineText = scopedLineTokens.getLineContent(); + let firstCharacterOffset: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstCharOffset; + let lastCharacterOffset: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstCharOffset; + let firstTokenIndex: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstTokenIndex; + let lastTokenIndex: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstTokenIndex; + let line: string; + + if (opts.isStart) { + const scopedLineTextLength = scopedLineText.length; + firstCharacterOffset += opts.columnIndexWithinScope + 1; + lastCharacterOffset += opts.columnIndexWithinScope + scopedLineTextLength - opts.columnIndexWithinScope + 1; + firstTokenIndex += scopedLineTokens.findTokenIndexAtOffset(opts.columnIndexWithinScope) + 1; + lastTokenIndex += scopedLineTokens.findTokenIndexAtOffset(scopedLineTextLength - 1) + 1; + line = scopedLineText.substring(opts.columnIndexWithinScope); + } else { + lastCharacterOffset += opts.columnIndexWithinScope; + lastTokenIndex += scopedLineTokens.findTokenIndexAtOffset(opts.columnIndexWithinScope); + line = scopedLineText.substring(0, opts.columnIndexWithinScope); + } + const processedTokens = new ScopedLineTokens( + initialLineTokens, + languageId, + firstTokenIndex, + lastTokenIndex, + firstCharacterOffset, + lastCharacterOffset + ); + const processedLine = this.getProcessedLineForLineAndTokens(languageId, line, processedTokens); + + console.log('getStrippedScopedLineTextFor'); + console.log('opts : ', opts); + console.log('initialLineTokens : ', initialLineTokens); + console.log('scopedLineTokens : ', scopedLineTokens); + console.log('firstCharacterOffset : ', firstCharacterOffset); + console.log('lastCharacterOffset : ', lastCharacterOffset); + console.log('firstTokenIndex : ', firstTokenIndex); + console.log('lastTokenIndex : ', lastTokenIndex); + console.log('processedLine : ', processedLine); + + return { line, processedLine } + } +} + +export const withinEmbeddedLanguage = (lineTokens: LineTokens, scopedLineTokens: ScopedLineTokens) => { + return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; +}; diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 8fdfa17bb5132..247ae313ad76b 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -36,7 +36,7 @@ export class ScopedLineTokens { public readonly languageId: string; private readonly _actual: LineTokens; - private readonly _firstTokenIndex: number; + public readonly firstTokenIndex: number; private readonly _lastTokenIndex: number; public readonly firstCharOffset: number; private readonly _lastCharOffset: number; @@ -51,7 +51,7 @@ export class ScopedLineTokens { ) { this._actual = actual; this.languageId = languageId; - this._firstTokenIndex = firstTokenIndex; + this.firstTokenIndex = firstTokenIndex; this._lastTokenIndex = lastTokenIndex; this.firstCharOffset = firstCharOffset; this._lastCharOffset = lastCharOffset; @@ -67,16 +67,24 @@ export class ScopedLineTokens { return actualLineContent.substring(0, this.firstCharOffset + offset); } - public getTokenCount(): number { - return this._lastTokenIndex - this._firstTokenIndex; + public getCount(): number { + return this._lastTokenIndex - this.firstTokenIndex; } public findTokenIndexAtOffset(offset: number): number { - return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this._firstTokenIndex; + return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this.firstTokenIndex; } public getStandardTokenType(tokenIndex: number): StandardTokenType { - return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); + return this._actual.getStandardTokenType(tokenIndex + this.firstTokenIndex); + } + + public getStartOffset(tokenIndex: number): number { + return this._actual.getStartOffset(tokenIndex + this.firstTokenIndex) - this.firstCharOffset; + } + + public getEndOffset(tokenIndex: number): number { + return this._actual.getEndOffset(tokenIndex + this.firstTokenIndex) - this.firstCharOffset; } } From 331308296cd1386332c622bced4849210cb111db Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 17:18:02 +0200 Subject: [PATCH 02/47] polishing the code --- src/vs/editor/common/languages/autoIndent.ts | 100 ++++---- src/vs/editor/common/languages/enterAction.ts | 17 +- .../languages/lineProcessorForIndentation.ts | 215 ------------------ src/vs/editor/common/languages/supports.ts | 4 + .../common/languages/supports/indentRules.ts | 41 ++++ .../supports/indentationLineProcessor.ts | 197 ++++++++++++++++ 6 files changed, 294 insertions(+), 280 deletions(-) delete mode 100644 src/vs/editor/common/languages/lineProcessorForIndentation.ts create mode 100644 src/vs/editor/common/languages/supports/indentationLineProcessor.ts diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index a73c341535223..2612edfb84a45 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -7,12 +7,11 @@ import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { IndentAction } from 'vs/editor/common/languages/languageConfiguration'; -import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; -import { IndentConsts, IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; +import { IndentConsts, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; -import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { ScopedLineProcessorForIndentation, LineProcessorForIndentation, withinEmbeddedLanguage } from 'vs/editor/common/languages/lineProcessorForIndentation'; +import { IndentationContextProcessor, isWithinEmbeddedLanguage } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export interface IVirtualModel { tokenization: { @@ -36,7 +35,7 @@ export interface IIndentConverter { * 0: every line above are invalid * else: nearest preceding line of the same language */ -function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentRulesSupport: IndentRulesSupport) { +function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentRulesSupport: ProcessedIndentRulesSupport) { const languageId = model.tokenization.getLanguageIdAtPosition(lineNumber, 0); if (lineNumber > 1) { let lastLineNumber: number; @@ -47,7 +46,7 @@ function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentR return resultLineNumber; } const text = model.getLineContent(lastLineNumber); - if (indentRulesSupport.shouldIgnore(text) || /^\s+$/.test(text) || text === '') { + if (indentRulesSupport.shouldIgnore(lastLineNumber) || /^\s+$/.test(text) || text === '') { resultLineNumber = lastLineNumber; continue; } @@ -86,6 +85,7 @@ export function getInheritIndentForLine( if (!indentRulesSupport) { return null; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageConfigurationService); if (lineNumber <= 1) { return { @@ -107,7 +107,7 @@ export function getInheritIndentForLine( } } - const precedingUnIgnoredLine = getPrecedingValidLine(model, lineNumber, indentRulesSupport); + const precedingUnIgnoredLine = getPrecedingValidLine(model, lineNumber, processedIndentRulesSupport); if (precedingUnIgnoredLine < 0) { return null; } else if (precedingUnIgnoredLine < 1) { @@ -117,15 +117,15 @@ export function getInheritIndentForLine( }; } - const linesProcessor = new LineProcessorForIndentation(model, languageConfigurationService); - const precedingUnIgnoredLineContent = linesProcessor.getLine(precedingUnIgnoredLine); - if (indentRulesSupport.shouldIncrease(precedingUnIgnoredLineContent) || indentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLineContent)) { + if (processedIndentRulesSupport.shouldIncrease(precedingUnIgnoredLine) || processedIndentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLine)) { + const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); return { indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent), action: IndentAction.Indent, line: precedingUnIgnoredLine }; - } else if (indentRulesSupport.shouldDecrease(precedingUnIgnoredLineContent)) { + } else if (processedIndentRulesSupport.shouldDecrease(precedingUnIgnoredLine)) { + const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); return { indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent), action: null, @@ -152,7 +152,7 @@ export function getInheritIndentForLine( (previousLineIndentMetadata & IndentConsts.INDENT_NEXTLINE_MASK)) { let stopLine = 0; for (let i = previousLine - 1; i > 0; i--) { - if (indentRulesSupport.shouldIndentNextLine(linesProcessor.getLine(i))) { + if (processedIndentRulesSupport.shouldIndentNextLine(i)) { continue; } stopLine = i; @@ -175,17 +175,16 @@ export function getInheritIndentForLine( } else { // search from precedingUnIgnoredLine until we find one whose indent is not temporary for (let i = precedingUnIgnoredLine; i > 0; i--) { - const lineContent = linesProcessor.getLine(i); - if (indentRulesSupport.shouldIncrease(lineContent)) { + if (processedIndentRulesSupport.shouldIncrease(i)) { return { - indentation: strings.getLeadingWhitespace(lineContent), + indentation: strings.getLeadingWhitespace(model.getLineContent(i)), action: IndentAction.Indent, line: i }; - } else if (indentRulesSupport.shouldIndentNextLine(lineContent)) { + } else if (processedIndentRulesSupport.shouldIndentNextLine(i)) { let stopLine = 0; for (let j = i - 1; j > 0; j--) { - if (indentRulesSupport.shouldIndentNextLine(linesProcessor.getLine(i))) { + if (processedIndentRulesSupport.shouldIndentNextLine(i)) { continue; } stopLine = j; @@ -193,13 +192,13 @@ export function getInheritIndentForLine( } return { - indentation: strings.getLeadingWhitespace(linesProcessor.getLine(stopLine + 1)), + indentation: strings.getLeadingWhitespace(model.getLineContent(stopLine + 1)), action: null, line: stopLine + 1 }; - } else if (indentRulesSupport.shouldDecrease(lineContent)) { + } else if (processedIndentRulesSupport.shouldDecrease(i)) { return { - indentation: strings.getLeadingWhitespace(lineContent), + indentation: strings.getLeadingWhitespace(model.getLineContent(i)), action: null, line: i }; @@ -207,7 +206,7 @@ export function getInheritIndentForLine( } return { - indentation: strings.getLeadingWhitespace(linesProcessor.getLine(1)), + indentation: strings.getLeadingWhitespace(model.getLineContent(1)), action: null, line: 1 }; @@ -236,10 +235,8 @@ export function getGoodIndentForLine( if (!indentRulesSupport) { return null; } - + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageConfigurationService); const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); - const linesProcessor = new LineProcessorForIndentation(virtualModel, languageConfigurationService); - const lineContent = linesProcessor.getLine(lineNumber); if (indent) { const inheritLine = indent.line; @@ -247,16 +244,16 @@ export function getGoodIndentForLine( // Apply enter action as long as there are only whitespace lines between inherited line and this line. let shouldApplyEnterRules = true; for (let inBetweenLine = inheritLine; inBetweenLine < lineNumber - 1; inBetweenLine++) { - if (!/^\s*$/.test(linesProcessor.getLine(inBetweenLine))) { + if (!/^\s*$/.test(virtualModel.getLineContent(inBetweenLine))) { shouldApplyEnterRules = false; break; } } if (shouldApplyEnterRules) { - const enterResult = richEditSupport.onEnter(autoIndent, '', linesProcessor.getLine(inheritLine), ''); + const enterResult = richEditSupport.onEnter(autoIndent, '', virtualModel.getLineContent(inheritLine), ''); if (enterResult) { - let indentation = strings.getLeadingWhitespace(linesProcessor.getLine(inheritLine)); + let indentation = strings.getLeadingWhitespace(virtualModel.getLineContent(inheritLine)); if (enterResult.removeText) { indentation = indentation.substring(0, indentation.length - enterResult.removeText); @@ -271,7 +268,7 @@ export function getGoodIndentForLine( indentation = indentConverter.unshiftIndent(indentation); } - if (indentRulesSupport.shouldDecrease(lineContent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber)) { indentation = indentConverter.unshiftIndent(indentation); } @@ -284,7 +281,7 @@ export function getGoodIndentForLine( } } - if (indentRulesSupport.shouldDecrease(lineContent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber)) { if (indent.action === IndentAction.Indent) { return indent.indentation; } else { @@ -312,20 +309,12 @@ export function getIndentForEnter( return null; } model.tokenization.forceTokenization(range.startLineNumber); - const lineTokens = model.tokenization.getLineTokens(range.startLineNumber); - const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const processLines = new ScopedLineProcessorForIndentation(model, languageConfigurationService); - const embeddedLanguage = withinEmbeddedLanguage(lineTokens, scopedLineTokens); - - const processedBeforeLines = processLines.getProcessedLineBeforeRange(range, scopedLineTokens); - const beforeEnterText = processedBeforeLines.processedLine; - const processedAfterLines = processLines.getProcessedLineAfterRange(range, scopedLineTokens); - const afterEnterText = processedAfterLines.processedLine; - - console.log('beforeEnterText : ', beforeEnterText); - console.log('afterEnterText : ', afterEnterText); - - const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId).indentRulesSupport; + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); + const afterEnterText = processedContext.afterRangeText; + const beforeEnterText = processedContext.beforeRangeText; + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; if (!indentRulesSupport) { return null; } @@ -354,7 +343,8 @@ export function getIndentForEnter( } }; - const currentLineIndent = strings.getLeadingWhitespace(lineTokens.getLineContent()); + const embeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); + const currentLineIndent = strings.getLeadingWhitespace(model.getLineContent(range.startLineNumber)); const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); if (!afterEnterAction) { const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; @@ -395,26 +385,24 @@ export function getIndentActionForType( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } - const scopedLineTokens = getScopedLineTokens(model, range.startLineNumber, range.startColumn); - - if (scopedLineTokens.firstCharOffset) { + const isPositionWithinEmbeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); + if (isPositionWithinEmbeddedLanguage) { // this line has mixed languages and indentation rules will not work return null; } - const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId).indentRulesSupport; + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; if (!indentRulesSupport) { return null; } - const linesProcessor = new ScopedLineProcessorForIndentation(model, languageConfigurationService); - const beforeEnterResult = linesProcessor.getProcessedLineBeforeRange(range, scopedLineTokens); - const afterEnterResult = linesProcessor.getProcessedLineAfterRange(range, scopedLineTokens); - const fullText = beforeEnterResult.processedLine + afterEnterResult.processedLine; - const fullTextWithCharacter = beforeEnterResult.processedLine + ch + afterEnterResult.processedLine; - - console.log('fullText : ', fullText); - console.log('fullTextWithCharacter : ', fullTextWithCharacter); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); + const beforeEnterText = processedContext.beforeRangeText; + const afterEnterText = processedContext.afterRangeText; + const fullText = beforeEnterText + afterEnterText; + const fullTextWithCharacter = beforeEnterText + ch + afterEnterText; // If previous content already matches decreaseIndentPattern, it means indentation of this line should already be adjusted // Users might change the indentation by purpose and we should honor that instead of readjusting. diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index 64ce04bed2112..30a88d9d5bb53 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -7,8 +7,8 @@ import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { IndentAction, CompleteEnterAction } from 'vs/editor/common/languages/languageConfiguration'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; -import { getIndentationAtPosition, getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { ScopedLineProcessorForIndentation } from 'vs/editor/common/languages/lineProcessorForIndentation'; +import { getIndentationAtPosition, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export function getEnterAction( autoIndent: EditorAutoIndentStrategy, @@ -16,17 +16,16 @@ export function getEnterAction( range: Range, languageConfigurationService: ILanguageConfigurationService ): CompleteEnterAction | null { - const scopedLineTokens = getScopedLineTokens(model, range.startLineNumber, range.startColumn); - const richEditSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId); + model.tokenization.forceTokenization(range.startLineNumber); + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const richEditSupport = languageConfigurationService.getLanguageConfiguration(languageId); if (!richEditSupport) { return null; } - const processLines = new ScopedLineProcessorForIndentation(model, languageConfigurationService); - const beforeEnterText = processLines.getProcessedLineBeforeRange(range, scopedLineTokens).processedLine; - const afterEnterText = processLines.getProcessedLineAfterRange(range, scopedLineTokens).processedLine; - const previousLineText = processLines.getProcessedPreviousLine(range); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText); + const enterResult = richEditSupport.onEnter(autoIndent, processedContext.previousLineText, processedContext.beforeRangeText, processedContext.afterRangeText); if (!enterResult) { return null; } diff --git a/src/vs/editor/common/languages/lineProcessorForIndentation.ts b/src/vs/editor/common/languages/lineProcessorForIndentation.ts deleted file mode 100644 index 72776228e8652..0000000000000 --- a/src/vs/editor/common/languages/lineProcessorForIndentation.ts +++ /dev/null @@ -1,215 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { Range } from 'vs/editor/common/core/range'; -import { ITextModel } from 'vs/editor/common/model'; -import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { ScopedLineTokens } from 'vs/editor/common/languages/supports'; -import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; -import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; - -type LineTokensType = LineTokens | ScopedLineTokens; - -interface ProcessedLineForIndentation { - line: string; - processedLine: string; -} - -export class LineProcessorForIndentation { - - constructor( - protected readonly model: IVirtualModel, - protected readonly languageConfigurationService: ILanguageConfigurationService - ) { } - - getLine(lineNumber: number): string { - - const languageId = this.model.tokenization.getLanguageId(); - const lineContent = this.model.getLineContent(lineNumber); - const tokens = this.model.tokenization.getLineTokens(lineNumber); - const processedLine = this.getProcessedLineForLineAndTokens(languageId, lineContent, tokens); - - console.log('getStrippedLine'); - console.log('lineContent : ', lineContent); - console.log('processedLine : ', processedLine); - - return processedLine; - } - - protected getProcessedLineForLineAndTokens(languageId: string, line: string, tokens: LineTokensType): string { - - const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, line: string): { processedCharacterOffset: number, processedLine: string } => { - const result = removeBracketsFromTokenWithIndex(tokenIndex); - const processedCharacterOffset = characterOffset + result.tokenText.length - result.processedText.length; - const processedLine = line.substring(0, characterOffset + result.tokenStartCharacterOffset) + result.processedText + line.substring(characterOffset + result.tokenEndCharacterOffset); - return { processedCharacterOffset, processedLine }; - }; - const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { - const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); - const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); - const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); - const processedText = removeBracketsFromText(tokenText); - return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; - } - const removeBracketsFromText = (line: string): string => { - let processedLine = line; - openBrackets.forEach((bracket) => { - processedLine = processedLine.replace(bracket, '_'); - }); - closedBrackets.forEach((bracket) => { - processedLine = processedLine.replace(bracket, '_'); - }); - return processedLine; - } - - console.log('getStrippedLineForLineAndTokens'); - console.log('line : ', line); - console.log('languageID : ', languageId); - - const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; - if (!brackets) { - return line; - } - - const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); - const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - - let characterOffset = 0; - let processedLine = line; - - for (let i = 0; i < tokens.getCount(); i++) { - - console.log('i : ', i); - - const standardTokenType = tokens.getStandardTokenType(i); - if (standardTokenType === StandardTokenType.String - || standardTokenType === StandardTokenType.RegEx - || standardTokenType === StandardTokenType.Comment - ) { - const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); - characterOffset = result.processedCharacterOffset; - processedLine = result.processedLine; - } - } - return processedLine; - } -} - -export class ScopedLineProcessorForIndentation extends LineProcessorForIndentation { - - protected override readonly model: ITextModel; - - constructor( - model: ITextModel, - languageConfigurationService: ILanguageConfigurationService - ) { - super(model, languageConfigurationService); - this.model = model; - } - - getProcessedLineAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineForIndentation { - let columnIndexWithinScope: number; - let lineTokens: LineTokens; - // let afterScopedLineTokens: ScopedLineTokens - if (range.isEmpty()) { - columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; - lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - // afterScopedLineTokens = scopedLineTokens; - } else { - columnIndexWithinScope = range.endColumn - 1 - scopedLineTokens.firstCharOffset; - lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); - // afterScopedLineTokens = getScopedLineTokens(this.model, range.endLineNumber, range.endColumn); - } - return this.getProcessedScopedLine(lineTokens, scopedLineTokens, { isStart: true, columnIndexWithinScope }); - } - - getProcessedLineBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineForIndentation { - const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - let columnIndexWithinScope: number; - let scopedLineTokensOfInterest: ScopedLineTokens | LineTokens; - if (withinEmbeddedLanguage(lineTokens, scopedLineTokens)) { - // we are in the embeded language content - columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; - scopedLineTokensOfInterest = scopedLineTokens; - } else { - columnIndexWithinScope = range.startColumn - 1; - scopedLineTokensOfInterest = lineTokens; - } - return this.getProcessedScopedLine(lineTokens, scopedLineTokensOfInterest, { isStart: false, columnIndexWithinScope }); - } - - getProcessedPreviousLine(range: Range) { - let processedPreviousLine = ''; - const scopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber, range.startColumn); - const language = this.model.tokenization.getLanguageId(); - if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { - // This is not the first line and the entire line belongs to this mode - const previousLineScopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber - 1); - if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { - // The line above ends with text belonging to the same mode - const previousLine = previousLineScopedLineTokens.getLineContent(); - processedPreviousLine = this.getProcessedLineForLineAndTokens(language, previousLine, previousLineScopedLineTokens); - } - } - console.log('previousLineText : ', processedPreviousLine); - return processedPreviousLine; - } - - private getProcessedScopedLine(initialLineTokens: LineTokens, scopedLineTokens: LineTokensType, opts: { columnIndexWithinScope: number, isStart: boolean }): ProcessedLineForIndentation { - - let languageId: string; - if (scopedLineTokens instanceof LineTokens) { - languageId = scopedLineTokens.getLanguageId(0); - } else { - languageId = scopedLineTokens.languageId; - } - - const scopedLineText = scopedLineTokens.getLineContent(); - let firstCharacterOffset: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstCharOffset; - let lastCharacterOffset: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstCharOffset; - let firstTokenIndex: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstTokenIndex; - let lastTokenIndex: number = scopedLineTokens instanceof LineTokens ? 0 : scopedLineTokens.firstTokenIndex; - let line: string; - - if (opts.isStart) { - const scopedLineTextLength = scopedLineText.length; - firstCharacterOffset += opts.columnIndexWithinScope + 1; - lastCharacterOffset += opts.columnIndexWithinScope + scopedLineTextLength - opts.columnIndexWithinScope + 1; - firstTokenIndex += scopedLineTokens.findTokenIndexAtOffset(opts.columnIndexWithinScope) + 1; - lastTokenIndex += scopedLineTokens.findTokenIndexAtOffset(scopedLineTextLength - 1) + 1; - line = scopedLineText.substring(opts.columnIndexWithinScope); - } else { - lastCharacterOffset += opts.columnIndexWithinScope; - lastTokenIndex += scopedLineTokens.findTokenIndexAtOffset(opts.columnIndexWithinScope); - line = scopedLineText.substring(0, opts.columnIndexWithinScope); - } - const processedTokens = new ScopedLineTokens( - initialLineTokens, - languageId, - firstTokenIndex, - lastTokenIndex, - firstCharacterOffset, - lastCharacterOffset - ); - const processedLine = this.getProcessedLineForLineAndTokens(languageId, line, processedTokens); - - console.log('getStrippedScopedLineTextFor'); - console.log('opts : ', opts); - console.log('initialLineTokens : ', initialLineTokens); - console.log('scopedLineTokens : ', scopedLineTokens); - console.log('firstCharacterOffset : ', firstCharacterOffset); - console.log('lastCharacterOffset : ', lastCharacterOffset); - console.log('firstTokenIndex : ', firstTokenIndex); - console.log('lastTokenIndex : ', lastTokenIndex); - console.log('processedLine : ', processedLine); - - return { line, processedLine } - } -} - -export const withinEmbeddedLanguage = (lineTokens: LineTokens, scopedLineTokens: ScopedLineTokens) => { - return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; -}; diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 247ae313ad76b..f84b4607cda34 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -86,6 +86,10 @@ export class ScopedLineTokens { public getEndOffset(tokenIndex: number): number { return this._actual.getEndOffset(tokenIndex + this.firstTokenIndex) - this.firstCharOffset; } + + public getLanguageId(tokenIndex: number): string { + return this._actual.getLanguageId(tokenIndex + this.firstTokenIndex); + } } const enum IgnoreBracketsInTokens { diff --git a/src/vs/editor/common/languages/supports/indentRules.ts b/src/vs/editor/common/languages/supports/indentRules.ts index 6bb319dbd7e08..7e3cafcc2f53e 100644 --- a/src/vs/editor/common/languages/supports/indentRules.ts +++ b/src/vs/editor/common/languages/supports/indentRules.ts @@ -3,7 +3,10 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IndentationRule } from 'vs/editor/common/languages/languageConfiguration'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IndentationLineProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export const enum IndentConsts { INCREASE_MASK = 0b00000001, @@ -81,3 +84,41 @@ export class IndentRulesSupport { return ret; } } + +/** + * This class processes the lines (it removes the brackets of the given language configuration) before calling the {@link IndentRulesSupport} methods + */ +export class ProcessedIndentRulesSupport { + + private readonly _indentationLineProcessor: IndentationLineProcessor; + private readonly _indentRulesSupport: IndentRulesSupport; + + constructor( + model: IVirtualModel, + indentRulesSupport: IndentRulesSupport, + languageConfigurationService: ILanguageConfigurationService + ) { + this._indentRulesSupport = indentRulesSupport; + this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + } + + public shouldIncrease(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIncrease(processedLine); + } + + public shouldDecrease(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldDecrease(processedLine); + } + + public shouldIgnore(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIgnore(processedLine); + } + + public shouldIndentNextLine(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIndentNextLine(processedLine); + } +} diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts new file mode 100644 index 0000000000000..a10cbde6020fd --- /dev/null +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -0,0 +1,197 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Range } from 'vs/editor/common/core/range'; +import { ITextModel } from 'vs/editor/common/model'; +import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; +import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { Position } from 'vs/editor/common/core/position'; + +export class IndentationLineProcessor { + + constructor( + protected readonly model: IVirtualModel, + protected readonly languageConfigurationService: ILanguageConfigurationService + ) { } + + getProcessedLine(lineNumber: number): string { + const lineContent = this.model.getLineContent(lineNumber); + const tokens = this.model.tokenization.getLineTokens(lineNumber); + const processedLine = this.getProcessedLineForLineAndTokens(lineContent, tokens); + return processedLine; + } + + getProcessedLineForLineAndTokens(line: string, tokens: LineTokens | ScopedLineTokens): string { + + // Utility functions + const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, processedLine: string): { processedCharacterOffset: number, processedLine: string } => { + const result = removeBracketsFromTokenWithIndex(tokenIndex); + const processedCharacterOffset = characterOffset - (result.tokenText.length - result.processedText.length); + const lineBeforeCharacterOffset = processedLine.substring(0, characterOffset + result.tokenStartCharacterOffset); + const lineAfterCharacterOffset = processedLine.substring(characterOffset + result.tokenEndCharacterOffset); + const newProcessedLine = lineBeforeCharacterOffset + result.processedText + lineAfterCharacterOffset; + return { processedCharacterOffset, processedLine: newProcessedLine }; + }; + const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { + const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); + const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); + const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); + const processedText = removeBracketsFromText(tokenText); + return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; + } + const removeBracketsFromText = (line: string): string => { + let processedLine = line; + openBrackets.forEach((bracket) => { + const regex = new RegExp(escapeStringForRegex(bracket), "g"); + processedLine = processedLine.replace(regex, ''); + }); + closedBrackets.forEach((bracket) => { + const regex = new RegExp(escapeStringForRegex(bracket), "g"); + processedLine = processedLine.replace(regex, ''); + }); + return processedLine; + } + const escapeStringForRegex = (text: string): string => { + let res = ''; + for (const chr of text) { + res += escapeCharacterForRegex(chr); + } + return res; + }; + const escapeCharacterForRegex = (chr: string): string => { + return chr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + // Main code + const languageId = tokens.getLanguageId(0); + const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; + if (!brackets) { + return line; + } + const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); + const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); + + let characterOffset = 0; + let processedLine = line; + + for (let i = 0; i < tokens.getCount(); i++) { + const standardTokenType = tokens.getStandardTokenType(i); + if (standardTokenType === StandardTokenType.String + || standardTokenType === StandardTokenType.RegEx + || standardTokenType === StandardTokenType.Comment + ) { + const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); + characterOffset = result.processedCharacterOffset; + processedLine = result.processedLine; + } + } + return processedLine; + } +} + +export interface ProcessedIndentationContext { + beforeRangeText: string; + afterRangeText: string; + previousLineText: string; +} + +export class IndentationContextProcessor { + + private readonly model: ITextModel; + private readonly indentationLineProcessor: IndentationLineProcessor; + + constructor( + model: ITextModel, + languageConfigurationService: ILanguageConfigurationService + ) { + this.model = model; + this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + } + + getProcessedContextAroundRange(range: Range): ProcessedIndentationContext { + this.model.tokenization.forceTokenization(range.startLineNumber); + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); + const beforeRangeText = this._getProcessedTextBeforeRange(range, scopedLineTokens); + const afterRangeText = this._getProcessedTextAfterRange(range, scopedLineTokens); + const previousLineText = this._getProcessedPreviousLine(range, scopedLineTokens); + return { beforeRangeText, afterRangeText, previousLineText }; + } + + private _getProcessedTextAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): string { + let columnIndexWithinScope: number; + let lineTokens: LineTokens; + if (range.isEmpty()) { + columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; + lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + } else { + columnIndexWithinScope = range.endColumn - 1 - scopedLineTokens.firstCharOffset; + lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); + } + const scopedLineContent = scopedLineTokens.getLineContent(); + const scopedLineLength = scopedLineContent.length; + const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; + const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineLength + 1; + const firstTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(columnIndexWithinScope); + const lastTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(scopedLineLength - 1) + 1; + const line = scopedLineContent.substring(columnIndexWithinScope); + const languageId = scopedLineTokens.languageId; + const processedTokens = new ScopedLineTokens( + lineTokens, + languageId, + firstTokenIndex, + lastTokenIndex, + firstCharacterOffset, + lastCharacterOffset + ); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, processedTokens); + return processedLine; + } + + private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): string { + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; + const firstCharacterOffset = scopedLineTokens.firstCharOffset; + const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; + const firstTokenIndex = scopedLineTokens.firstTokenIndex; + const lastTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(columnIndexWithinScope) + 1; + const languageId = scopedLineTokens.languageId; + const processedTokens = new ScopedLineTokens( + lineTokens, + languageId, + firstTokenIndex, + lastTokenIndex, + firstCharacterOffset, + lastCharacterOffset + ); + const scopedLineContent = scopedLineTokens.getLineContent(); + const line = scopedLineContent.substring(0, columnIndexWithinScope); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, processedTokens); + return processedLine; + } + + private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens) { + let processedPreviousLine = ''; + if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { + // This is not the first line and the entire line belongs to this mode + const previousLineScopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber - 1); + if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { + // The line above ends with text belonging to the same mode + const previousLine = previousLineScopedLineTokens.getLineContent(); + processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousLineScopedLineTokens); + } + } + return processedPreviousLine; + } +} + +export function isWithinEmbeddedLanguage(model: ITextModel, position: Position): boolean { + const lineTokens = model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; +}; From cc8ea289d183c4681c190fde0297a87e630b2553 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 17:20:01 +0200 Subject: [PATCH 03/47] adding code --- src/vs/editor/common/languages/autoIndent.ts | 13 ++++++------- .../languages/supports/indentationLineProcessor.ts | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 2612edfb84a45..5d03600458cd9 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -308,18 +308,17 @@ export function getIndentForEnter( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } + const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentRulesSupport) { + return null; + } + model.tokenization.forceTokenization(range.startLineNumber); const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); const afterEnterText = processedContext.afterRangeText; const beforeEnterText = processedContext.beforeRangeText; const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); - const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; - if (!indentRulesSupport) { - return null; - } - - const beforeEnterResult = beforeEnterText; const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterText); const virtualModel: IVirtualModel = { @@ -336,7 +335,7 @@ export function getIndentForEnter( }, getLineContent: (lineNumber: number) => { if (lineNumber === range.startLineNumber) { - return beforeEnterResult; + return beforeEnterText; } else { return model.getLineContent(lineNumber); } diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index a10cbde6020fd..fff551780ccc1 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -63,8 +63,8 @@ export class IndentationLineProcessor { } return res; }; - const escapeCharacterForRegex = (chr: string): string => { - return chr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const escapeCharacterForRegex = (character: string): string => { + return character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); } // Main code From 42e9b121319522d7b2cc43bfd45c4e7b3c11b939 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 17:20:29 +0200 Subject: [PATCH 04/47] adding the language --- src/vs/editor/common/languages/autoIndent.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 5d03600458cd9..aef8f49863968 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -308,6 +308,7 @@ export function getIndentForEnter( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; if (!indentRulesSupport) { return null; @@ -318,7 +319,6 @@ export function getIndentForEnter( const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); const afterEnterText = processedContext.afterRangeText; const beforeEnterText = processedContext.beforeRangeText; - const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterText); const virtualModel: IVirtualModel = { From 31bab160af55813efd1d2dbddc711b77c2b5a9cc Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 17:28:29 +0200 Subject: [PATCH 05/47] reshuffling the code to avoid cyclic dependency --- .../languageConfigurationRegistry.ts | 8 --- .../common/languages/supports/indentRules.ts | 41 ---------------- .../supports/indentationLineProcessor.ts | 49 +++++++++++++++++-- 3 files changed, 46 insertions(+), 52 deletions(-) diff --git a/src/vs/editor/common/languages/languageConfigurationRegistry.ts b/src/vs/editor/common/languages/languageConfigurationRegistry.ts index d8598afe6fd97..7ff5ddec8a63a 100644 --- a/src/vs/editor/common/languages/languageConfigurationRegistry.ts +++ b/src/vs/editor/common/languages/languageConfigurationRegistry.ts @@ -9,7 +9,6 @@ import * as strings from 'vs/base/common/strings'; import { ITextModel } from 'vs/editor/common/model'; import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/core/wordHelper'; import { EnterAction, FoldingRules, IAutoClosingPair, IndentationRule, LanguageConfiguration, AutoClosingPairs, CharacterPair, ExplicitLanguageConfiguration } from 'vs/editor/common/languages/languageConfiguration'; -import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; import { CharacterPairSupport } from 'vs/editor/common/languages/supports/characterPair'; import { BracketElectricCharacterSupport } from 'vs/editor/common/languages/supports/electricCharacter'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; @@ -181,13 +180,6 @@ export function getIndentationAtPosition(model: ITextModel, lineNumber: number, return indentation; } -export function getScopedLineTokens(model: ITextModel, lineNumber: number, columnNumber?: number): ScopedLineTokens { - model.tokenization.forceTokenization(lineNumber); - const lineTokens = model.tokenization.getLineTokens(lineNumber); - const column = (typeof columnNumber === 'undefined' ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1); - return createScopedLineTokens(lineTokens, column); -} - class ComposedLanguageConfiguration { private readonly _entries: LanguageConfigurationContribution[]; private _order: number; diff --git a/src/vs/editor/common/languages/supports/indentRules.ts b/src/vs/editor/common/languages/supports/indentRules.ts index 7e3cafcc2f53e..6bb319dbd7e08 100644 --- a/src/vs/editor/common/languages/supports/indentRules.ts +++ b/src/vs/editor/common/languages/supports/indentRules.ts @@ -3,10 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IndentationRule } from 'vs/editor/common/languages/languageConfiguration'; -import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { IndentationLineProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export const enum IndentConsts { INCREASE_MASK = 0b00000001, @@ -84,41 +81,3 @@ export class IndentRulesSupport { return ret; } } - -/** - * This class processes the lines (it removes the brackets of the given language configuration) before calling the {@link IndentRulesSupport} methods - */ -export class ProcessedIndentRulesSupport { - - private readonly _indentationLineProcessor: IndentationLineProcessor; - private readonly _indentRulesSupport: IndentRulesSupport; - - constructor( - model: IVirtualModel, - indentRulesSupport: IndentRulesSupport, - languageConfigurationService: ILanguageConfigurationService - ) { - this._indentRulesSupport = indentRulesSupport; - this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); - } - - public shouldIncrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) - return this._indentRulesSupport.shouldIncrease(processedLine); - } - - public shouldDecrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) - return this._indentRulesSupport.shouldDecrease(processedLine); - } - - public shouldIgnore(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) - return this._indentRulesSupport.shouldIgnore(processedLine); - } - - public shouldIndentNextLine(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) - return this._indentRulesSupport.shouldIndentNextLine(processedLine); - } -} diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index fff551780ccc1..600705fd07c2a 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -5,12 +5,51 @@ import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; -import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; +import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; + +/** + * This class processes the lines (it removes the brackets of the given language configuration) before calling the {@link IndentRulesSupport} methods + */ +export class ProcessedIndentRulesSupport { + + private readonly _indentationLineProcessor: IndentationLineProcessor; + private readonly _indentRulesSupport: IndentRulesSupport; + + constructor( + model: IVirtualModel, + indentRulesSupport: IndentRulesSupport, + languageConfigurationService: ILanguageConfigurationService + ) { + this._indentRulesSupport = indentRulesSupport; + this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + } + + public shouldIncrease(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIncrease(processedLine); + } + + public shouldDecrease(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldDecrease(processedLine); + } + + public shouldIgnore(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIgnore(processedLine); + } + + public shouldIndentNextLine(lineNumber: number): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + return this._indentRulesSupport.shouldIndentNextLine(processedLine); + } +} export class IndentationLineProcessor { @@ -94,7 +133,7 @@ export class IndentationLineProcessor { } } -export interface ProcessedIndentationContext { +interface ProcessedIndentationContext { beforeRangeText: string; afterRangeText: string; previousLineText: string; @@ -179,7 +218,11 @@ export class IndentationContextProcessor { let processedPreviousLine = ''; if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { // This is not the first line and the entire line belongs to this mode - const previousLineScopedLineTokens = getScopedLineTokens(this.model, range.startLineNumber - 1); + const previousLineNumber = range.startLineNumber - 1; + this.model.tokenization.forceTokenization(previousLineNumber); + const lineTokens = this.model.tokenization.getLineTokens(previousLineNumber); + const column = this.model.getLineMaxColumn(previousLineNumber) - 1; + const previousLineScopedLineTokens = createScopedLineTokens(lineTokens, column); if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { // The line above ends with text belonging to the same mode const previousLine = previousLineScopedLineTokens.getLineContent(); From 5f15946730db360ae5a0410900814f1dd9aed73e Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 17:40:10 +0200 Subject: [PATCH 06/47] polihsing code --- src/vs/editor/common/languages/autoIndent.ts | 12 +- .../supports/indentationLineProcessor.ts | 198 +++++++++--------- 2 files changed, 111 insertions(+), 99 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index aef8f49863968..4dd7ed09ed8ef 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -7,11 +7,13 @@ import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { IndentAction } from 'vs/editor/common/languages/languageConfiguration'; -import { IndentConsts, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; +import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { IndentationContextProcessor, isWithinEmbeddedLanguage } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; +import { Position } from 'vs/editor/common/core/position'; export interface IVirtualModel { tokenization: { @@ -438,3 +440,9 @@ export function getIndentMetadata( } return indentRulesSupport.getIndentMetadata(model.getLineContent(lineNumber)); } + +export function isWithinEmbeddedLanguage(model: ITextModel, position: Position): boolean { + const lineTokens = model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; +}; diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 600705fd07c2a..26ebad5f86501 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -7,14 +7,15 @@ import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; -import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { Position } from 'vs/editor/common/core/position'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; /** - * This class processes the lines (it removes the brackets of the given language configuration) before calling the {@link IndentRulesSupport} methods + * This class is a wrapper class around {@link IndentRulesSupport}. + * It processes the lines by removing the language configuration brackets from the regex, string and comment tokens. + * It then calls into the {@link IndentRulesSupport} to validate the indentation conditions. */ export class ProcessedIndentRulesSupport { @@ -51,94 +52,13 @@ export class ProcessedIndentRulesSupport { } } -export class IndentationLineProcessor { - - constructor( - protected readonly model: IVirtualModel, - protected readonly languageConfigurationService: ILanguageConfigurationService - ) { } - - getProcessedLine(lineNumber: number): string { - const lineContent = this.model.getLineContent(lineNumber); - const tokens = this.model.tokenization.getLineTokens(lineNumber); - const processedLine = this.getProcessedLineForLineAndTokens(lineContent, tokens); - return processedLine; - } - - getProcessedLineForLineAndTokens(line: string, tokens: LineTokens | ScopedLineTokens): string { - - // Utility functions - const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, processedLine: string): { processedCharacterOffset: number, processedLine: string } => { - const result = removeBracketsFromTokenWithIndex(tokenIndex); - const processedCharacterOffset = characterOffset - (result.tokenText.length - result.processedText.length); - const lineBeforeCharacterOffset = processedLine.substring(0, characterOffset + result.tokenStartCharacterOffset); - const lineAfterCharacterOffset = processedLine.substring(characterOffset + result.tokenEndCharacterOffset); - const newProcessedLine = lineBeforeCharacterOffset + result.processedText + lineAfterCharacterOffset; - return { processedCharacterOffset, processedLine: newProcessedLine }; - }; - const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { - const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); - const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); - const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); - const processedText = removeBracketsFromText(tokenText); - return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; - } - const removeBracketsFromText = (line: string): string => { - let processedLine = line; - openBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), "g"); - processedLine = processedLine.replace(regex, ''); - }); - closedBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), "g"); - processedLine = processedLine.replace(regex, ''); - }); - return processedLine; - } - const escapeStringForRegex = (text: string): string => { - let res = ''; - for (const chr of text) { - res += escapeCharacterForRegex(chr); - } - return res; - }; - const escapeCharacterForRegex = (character: string): string => { - return character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - } - - // Main code - const languageId = tokens.getLanguageId(0); - const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; - if (!brackets) { - return line; - } - const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); - const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - - let characterOffset = 0; - let processedLine = line; - - for (let i = 0; i < tokens.getCount(); i++) { - const standardTokenType = tokens.getStandardTokenType(i); - if (standardTokenType === StandardTokenType.String - || standardTokenType === StandardTokenType.RegEx - || standardTokenType === StandardTokenType.Comment - ) { - const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); - characterOffset = result.processedCharacterOffset; - processedLine = result.processedLine; - } - } - return processedLine; - } -} - -interface ProcessedIndentationContext { - beforeRangeText: string; - afterRangeText: string; - previousLineText: string; -} - +/** + * This class fetches the processed text around a range which can be used for indentation evaluation. + * It returns: + * - The processed text before the given range and on the same start line + * - The processed text after the given range and on the same end line + * - The processed text on the previous line + */ export class IndentationContextProcessor { private readonly model: ITextModel; @@ -152,7 +72,11 @@ export class IndentationContextProcessor { this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } - getProcessedContextAroundRange(range: Range): ProcessedIndentationContext { + getProcessedContextAroundRange(range: Range): { + beforeRangeText: string; + afterRangeText: string; + previousLineText: string; + } { this.model.tokenization.forceTokenization(range.startLineNumber); const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); @@ -233,8 +157,88 @@ export class IndentationContextProcessor { } } -export function isWithinEmbeddedLanguage(model: ITextModel, position: Position): boolean { - const lineTokens = model.tokenization.getLineTokens(position.lineNumber); - const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); - return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; -}; +/** + * This class performs the actual processing of the indentation lines. + * The brackets of the language configuration are removed from the regex, string and comment tokens. + */ +class IndentationLineProcessor { + + constructor( + protected readonly model: IVirtualModel, + protected readonly languageConfigurationService: ILanguageConfigurationService + ) { } + + getProcessedLine(lineNumber: number): string { + const lineContent = this.model.getLineContent(lineNumber); + const tokens = this.model.tokenization.getLineTokens(lineNumber); + const processedLine = this.getProcessedLineForLineAndTokens(lineContent, tokens); + return processedLine; + } + + getProcessedLineForLineAndTokens(line: string, tokens: LineTokens | ScopedLineTokens): string { + + // Utility functions + const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, processedLine: string): { processedCharacterOffset: number, processedLine: string } => { + const result = removeBracketsFromTokenWithIndex(tokenIndex); + const processedCharacterOffset = characterOffset - (result.tokenText.length - result.processedText.length); + const lineBeforeCharacterOffset = processedLine.substring(0, characterOffset + result.tokenStartCharacterOffset); + const lineAfterCharacterOffset = processedLine.substring(characterOffset + result.tokenEndCharacterOffset); + const newProcessedLine = lineBeforeCharacterOffset + result.processedText + lineAfterCharacterOffset; + return { processedCharacterOffset, processedLine: newProcessedLine }; + }; + const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { + const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); + const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); + const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); + const processedText = removeBracketsFromText(tokenText); + return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; + } + const removeBracketsFromText = (line: string): string => { + let processedLine = line; + openBrackets.forEach((bracket) => { + const regex = new RegExp(escapeStringForRegex(bracket), "g"); + processedLine = processedLine.replace(regex, ''); + }); + closedBrackets.forEach((bracket) => { + const regex = new RegExp(escapeStringForRegex(bracket), "g"); + processedLine = processedLine.replace(regex, ''); + }); + return processedLine; + } + const escapeStringForRegex = (text: string): string => { + let res = ''; + for (const chr of text) { + res += escapeCharacterForRegex(chr); + } + return res; + }; + const escapeCharacterForRegex = (character: string): string => { + return character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + } + + // Main code + const languageId = tokens.getLanguageId(0); + const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; + if (!brackets) { + return line; + } + const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); + const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); + + let characterOffset = 0; + let processedLine = line; + + for (let i = 0; i < tokens.getCount(); i++) { + const standardTokenType = tokens.getStandardTokenType(i); + if (standardTokenType === StandardTokenType.String + || standardTokenType === StandardTokenType.RegEx + || standardTokenType === StandardTokenType.Comment + ) { + const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); + characterOffset = result.processedCharacterOffset; + processedLine = result.processedLine; + } + } + return processedLine; + } +} From 3b93b8ee50e6334c108d4d156bb09e207b2b4354 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 18 Apr 2024 18:08:29 +0200 Subject: [PATCH 07/47] uncommenting tests --- src/vs/editor/common/languages/autoIndent.ts | 10 ++++---- .../test/browser/indentation.test.ts | 24 ++++++++++++------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 4dd7ed09ed8ef..c64b254d761f2 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -400,14 +400,14 @@ export function getIndentActionForType( const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const beforeEnterText = processedContext.beforeRangeText; - const afterEnterText = processedContext.afterRangeText; - const fullText = beforeEnterText + afterEnterText; - const fullTextWithCharacter = beforeEnterText + ch + afterEnterText; + const beforeRangeText = processedContext.beforeRangeText; + const afterRangeText = processedContext.afterRangeText; + const textAroundRange = beforeRangeText + afterRangeText; + const textAroundRangeWithCharacter = beforeRangeText + ch + afterRangeText; // If previous content already matches decreaseIndentPattern, it means indentation of this line should already be adjusted // Users might change the indentation by purpose and we should honor that instead of readjusting. - if (!indentRulesSupport.shouldDecrease(fullText) && indentRulesSupport.shouldDecrease(fullTextWithCharacter)) { + if (!indentRulesSupport.shouldDecrease(textAroundRange) && indentRulesSupport.shouldDecrease(textAroundRangeWithCharacter)) { // after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner. // 1. Get inherited indent action const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageConfigurationService); diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index b1854502afb36..d66f65df07524 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -986,9 +986,7 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { }); }); - // Failing tests... - - test.skip('issue #208232: incorrect indentation inside of comments', () => { + test('issue #208232: incorrect indentation inside of comments', () => { // https://github.com/microsoft/vscode/issues/208232 @@ -1000,8 +998,13 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { disposables.add(model); withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { - + const tokens = [ + [{ startIndex: 0, value: 1 }], + [{ startIndex: 0, value: 1 }], + [{ startIndex: 0, value: 1 }] + ]; registerLanguage(instantiationService, languageId, Language.TypeScript, disposables); + registerTokens(instantiationService, tokens, languageId, disposables); editor.setSelection(new Selection(2, 23, 2, 23)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ @@ -1013,6 +1016,8 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { }); }); + // Failing tests... + test.skip('issue #43244: indent after equal sign is detected', () => { // https://github.com/microsoft/vscode/issues/43244 @@ -1381,20 +1386,21 @@ suite('Auto Indent On Type - PHP', () => { assert.ok(true); }); - test.skip('issue #199050: should not indent after { detected in a string', () => { + test('issue #199050: should not indent after { detected in a string', () => { // https://github.com/microsoft/vscode/issues/199050 - const model = createTextModel("$phrase = preg_replace('#(\{1|%s).*#su', '', $phrase);", languageId, {}); + const model = createTextModel("preg_replace('{');", languageId, {}); disposables.add(model); withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { - + const tokens = [[{ startIndex: 0, value: 0 }, { startIndex: 13, value: 2 }, { startIndex: 16, value: 0 }]]; + registerTokens(instantiationService, tokens, languageId, disposables); registerLanguage(instantiationService, languageId, Language.PHP, disposables); - editor.setSelection(new Selection(1, 54, 1, 54)); + editor.setSelection(new Selection(1, 19, 1, 19)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ - "$phrase = preg_replace('#(\{1|%s).*#su', '', $phrase);", + "preg_replace('{');", "" ].join('\n')); }); From c5335b7b48c2e3ebf37d2cd89204b43e1603e20f Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 12:03:07 +0200 Subject: [PATCH 08/47] also adopting the indentation rules within the reindentation operation --- .../supports/indentationLineProcessor.ts | 36 +++++++++-- src/vs/editor/common/tokens/lineTokens.ts | 9 +++ .../contrib/indentation/common/indentation.ts | 49 +++++---------- .../codeEditor/test/node/autoindent.test.ts | 63 ++++++++++++++++++- 4 files changed, 116 insertions(+), 41 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 26ebad5f86501..dce6bc5d07f72 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; @@ -21,35 +22,62 @@ export class ProcessedIndentRulesSupport { private readonly _indentationLineProcessor: IndentationLineProcessor; private readonly _indentRulesSupport: IndentRulesSupport; + private readonly _model: IVirtualModel; constructor( model: IVirtualModel, indentRulesSupport: IndentRulesSupport, languageConfigurationService: ILanguageConfigurationService ) { + this._model = model; this._indentRulesSupport = indentRulesSupport; this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } public shouldIncrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); return this._indentRulesSupport.shouldIncrease(processedLine); } public shouldDecrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); return this._indentRulesSupport.shouldDecrease(processedLine); } public shouldIgnore(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); return this._indentRulesSupport.shouldIgnore(processedLine); } public shouldIndentNextLine(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber) + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); return this._indentRulesSupport.shouldIndentNextLine(processedLine); } + + public shouldIncreaseAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { + const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); + return this._indentRulesSupport.shouldIncrease(processedLine); + } + + public shouldIndentNextLineAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { + const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); + return this._indentRulesSupport.shouldIndentNextLine(processedLine); + } + + public shouldDecreaseAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { + const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); + return this._indentRulesSupport.shouldDecrease(processedLine); + } + + private _adjustIndentationForLineAndProcess(lineNumber: number, newIndentation: string): string { + const currentLine = this._model.getLineContent(lineNumber); + const currentIndentation = strings.getLeadingWhitespace(currentLine); + const currentTokens = this._model.tokenization.getLineTokens(lineNumber); + const indentationDifference = newIndentation.length - currentIndentation.length; + const newLine = newIndentation + currentLine.substring(currentIndentation.length);; + const newTokens = currentTokens.shiftTokenOffsetsBy(indentationDifference, newLine); + return this._indentationLineProcessor.getProcessedLineForLineAndTokens(newLine, newTokens); + } } /** diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index bdcda7cb180c2..da11aa7aa297f 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -181,6 +181,15 @@ export class LineTokens implements IViewLineTokens { return low; } + public shiftTokenOffsetsBy(delta: number, newText: string): LineTokens { + const newTokens = new Array(); + for (let i = 0; i < this.getCount(); i++) { + newTokens.push(this.getEndOffset(i) + delta); + newTokens.push(this.getMetadata(i)); + } + return new LineTokens(new Uint32Array(newTokens), newText, this._languageIdCodec); + } + /** * @pure * @param insertTokens Must be sorted by offset. diff --git a/src/vs/editor/contrib/indentation/common/indentation.ts b/src/vs/editor/contrib/indentation/common/indentation.ts index 760a14919fb88..32fb40a5f613c 100644 --- a/src/vs/editor/contrib/indentation/common/indentation.ts +++ b/src/vs/editor/contrib/indentation/common/indentation.ts @@ -9,29 +9,26 @@ import { EditOperation, ISingleEditOperation } from 'vs/editor/common/core/editO import { normalizeIndentation } from 'vs/editor/common/core/indentation'; import { Selection } from 'vs/editor/common/core/selection'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { ITextModel } from 'vs/editor/common/model'; -export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number, inheritedIndent?: string): ISingleEditOperation[] { +export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { if (model.getLineCount() === 1 && model.getLineMaxColumn(1) === 1) { // Model is empty return []; } - const indentationRules = languageConfigurationService.getLanguageConfiguration(model.getLanguageId()).indentationRules; - if (!indentationRules) { + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(model.getLanguageId()).indentRulesSupport; + if (!indentationRulesSupport) { return []; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); endLineNumber = Math.min(endLineNumber, model.getLineCount()); // Skip `unIndentedLinePattern` lines while (startLineNumber <= endLineNumber) { - if (!indentationRules.unIndentedLinePattern) { - break; - } - - const text = model.getLineContent(startLineNumber); - if (!indentationRules.unIndentedLinePattern.test(text)) { + if (!processedIndentRulesSupport.shouldIgnore(startLineNumber)) { break; } @@ -54,37 +51,19 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati const indentEdits: ISingleEditOperation[] = []; // indentation being passed to lines below - let globalIndent: string; // Calculate indentation for the first line // If there is no passed-in indentation, we use the indentation of the first line as base. const currentLineText = model.getLineContent(startLineNumber); - let adjustedLineContent = currentLineText; - if (inheritedIndent !== undefined && inheritedIndent !== null) { - globalIndent = inheritedIndent; - const oldIndentation = strings.getLeadingWhitespace(currentLineText); - - adjustedLineContent = globalIndent + currentLineText.substring(oldIndentation.length); - if (indentationRules.decreaseIndentPattern && indentationRules.decreaseIndentPattern.test(adjustedLineContent)) { - globalIndent = unshiftIndent(globalIndent); - adjustedLineContent = globalIndent + currentLineText.substring(oldIndentation.length); - - } - if (currentLineText !== adjustedLineContent) { - indentEdits.push(EditOperation.replaceMove(new Selection(startLineNumber, 1, startLineNumber, oldIndentation.length + 1), normalizeIndentation(globalIndent, indentSize, insertSpaces))); - } - } else { - globalIndent = strings.getLeadingWhitespace(currentLineText); - } - + let globalIndent = strings.getLeadingWhitespace(currentLineText); // idealIndentForNextLine doesn't equal globalIndent when there is a line matching `indentNextLinePattern`. let idealIndentForNextLine: string = globalIndent; - if (indentationRules.increaseIndentPattern && indentationRules.increaseIndentPattern.test(adjustedLineContent)) { + if (processedIndentRulesSupport.shouldIncrease(startLineNumber)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); globalIndent = shiftIndent(globalIndent); } - else if (indentationRules.indentNextLinePattern && indentationRules.indentNextLinePattern.test(adjustedLineContent)) { + else if (processedIndentRulesSupport.shouldIndentNextLine(startLineNumber)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); } @@ -94,9 +73,9 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) { const text = model.getLineContent(lineNumber); const oldIndentation = strings.getLeadingWhitespace(text); - const adjustedLineContent = idealIndentForNextLine + text.substring(oldIndentation.length); + const currentIdealIndent = idealIndentForNextLine; - if (indentationRules.decreaseIndentPattern && indentationRules.decreaseIndentPattern.test(adjustedLineContent)) { + if (processedIndentRulesSupport.shouldDecreaseAfterSettingIndentation(lineNumber, currentIdealIndent)) { idealIndentForNextLine = unshiftIndent(idealIndentForNextLine); globalIndent = unshiftIndent(globalIndent); } @@ -106,14 +85,14 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati } // calculate idealIndentForNextLine - if (indentationRules.unIndentedLinePattern && indentationRules.unIndentedLinePattern.test(text)) { + if (processedIndentRulesSupport.shouldIgnore(lineNumber)) { // In reindent phase, if the line matches `unIndentedLinePattern` we inherit indentation from above lines // but don't change globalIndent and idealIndentForNextLine. continue; - } else if (indentationRules.increaseIndentPattern && indentationRules.increaseIndentPattern.test(adjustedLineContent)) { + } else if (processedIndentRulesSupport.shouldIncreaseAfterSettingIndentation(lineNumber, currentIdealIndent)) { globalIndent = shiftIndent(globalIndent); idealIndentForNextLine = globalIndent; - } else if (indentationRules.indentNextLinePattern && indentationRules.indentNextLinePattern.test(adjustedLineContent)) { + } else if (processedIndentRulesSupport.shouldIndentNextLineAfterSettingIndentation(lineNumber, currentIdealIndent)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); } else { idealIndentForNextLine = globalIndent; diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index cc9316468bc41..2b77dcc17b5e5 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -18,6 +18,10 @@ import { IRange } from 'vs/editor/common/core/range'; import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { trimTrailingWhitespace } from 'vs/editor/common/commands/trimTrailingWhitespaceCommand'; import { execSync } from 'child_process'; +import { ILanguageService } from 'vs/editor/common/languages/language'; +import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; +import { NullState } from 'vs/editor/common/languages/nullTokenize'; +import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; function getIRange(range: IRange): IRange { return { @@ -32,7 +36,11 @@ const enum LanguageId { TypeScript = 'ts-test' } -function registerLanguage(languageConfigurationService: ILanguageConfigurationService, languageId: LanguageId): IDisposable { +function registerLanguage(languageService: ILanguageService, languageId: LanguageId): IDisposable { + return languageService.registerLanguage({ id: languageId }); +} + +function registerLanguageConfiguration(languageConfigurationService: ILanguageConfigurationService, languageId: LanguageId): IDisposable { let configPath: string; switch (languageId) { case LanguageId.TypeScript: @@ -47,6 +55,27 @@ function registerLanguage(languageConfigurationService: ILanguageConfigurationSe return languageConfigurationService.register(languageId, languageConfig); } +function registerTokenizationSupport(languageService: ILanguageService, tokens: { startIndex: number; value: number }[][], languageId: LanguageId): IDisposable { + let lineIndex = 0; + const tokenizationSupport: ITokenizationSupport = { + getInitialState: () => NullState, + tokenize: undefined!, + tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { + const tokensOnLine = tokens[lineIndex++]; + const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); + const result = new Uint32Array(2 * tokensOnLine.length); + for (let i = 0; i < tokensOnLine.length; i++) { + result[2 * i] = tokensOnLine[i].startIndex; + result[2 * i + 1] = + ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); + } + return new EncodedTokenizationResult(result, state); + } + }; + return TokenizationRegistry.register(languageId, tokenizationSupport); +} + suite('Auto-Reindentation - TypeScript/JavaScript', () => { const languageId = LanguageId.TypeScript; @@ -54,12 +83,16 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { let disposables: DisposableStore; let instantiationService: TestInstantiationService; let languageConfigurationService: ILanguageConfigurationService; + let languageService: ILanguageService; setup(() => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); + disposables.add(instantiationService); + languageService = instantiationService.get(ILanguageService); + disposables.add(registerLanguage(languageService, languageId)); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); - disposables.add(registerLanguage(languageConfigurationService, languageId)); + disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); }); teardown(() => { @@ -160,7 +193,20 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const foo = `{`;', ' ', ].join('\n'); + const tokens = [ + [ + { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, + { startIndex: 9, value: 0 }, { startIndex: 10, value: 0 }, { startIndex: 11, value: 0 }, + { startIndex: 12, value: 2 }, { startIndex: 13, value: 2 }, { startIndex: 14, value: 2 }, + { startIndex: 15, value: 0 }, { startIndex: 16, value: 0 }, + ], + [{ startIndex: 0, value: 0 }, { startIndex: 1, value: 0 }] + ]; + disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); + model.tokenization.forceTokenization(1); + model.tokenization.forceTokenization(2); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; @@ -258,7 +304,20 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const r = /{/;', ' ', ].join('\n'); + const tokens = [ + [ + { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, + { startIndex: 7, value: 0 }, { startIndex: 8, value: 0 }, { startIndex: 9, value: 3 }, + { startIndex: 10, value: 3 }, { startIndex: 11, value: 3 }, { startIndex: 12, value: 3 }, + { startIndex: 13, value: 0 }, { startIndex: 14, value: 0 } + ], + [{ startIndex: 0, value: 0 }, { startIndex: 1, value: 0 }] + ]; + disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); + model.tokenization.forceTokenization(1); + model.tokenization.forceTokenization(2); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; From 50bb21fc60661e5149d27134a773223269c6586b Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 12:35:21 +0200 Subject: [PATCH 09/47] using instead the sliced line tokens instead of the scoped line tokens --- .../common/languages/languageConfiguration.ts | 2 +- src/vs/editor/common/languages/supports.ts | 24 +++-------- .../supports/indentationLineProcessor.ts | 41 ++++++------------- src/vs/editor/common/tokens/lineTokens.ts | 11 +++++ .../editor/test/common/core/testLineToken.ts | 17 +++++++- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/vs/editor/common/languages/languageConfiguration.ts b/src/vs/editor/common/languages/languageConfiguration.ts index 7ecf1e1f71c97..69b852e407c4f 100644 --- a/src/vs/editor/common/languages/languageConfiguration.ts +++ b/src/vs/editor/common/languages/languageConfiguration.ts @@ -321,7 +321,7 @@ export class StandardAutoClosingPairConditional { public shouldAutoClose(context: ScopedLineTokens, column: number): boolean { // Always complete on empty line - if (context.getCount() === 0) { + if (context.getTokenCount() === 0) { return true; } diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index f84b4607cda34..8fdfa17bb5132 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -36,7 +36,7 @@ export class ScopedLineTokens { public readonly languageId: string; private readonly _actual: LineTokens; - public readonly firstTokenIndex: number; + private readonly _firstTokenIndex: number; private readonly _lastTokenIndex: number; public readonly firstCharOffset: number; private readonly _lastCharOffset: number; @@ -51,7 +51,7 @@ export class ScopedLineTokens { ) { this._actual = actual; this.languageId = languageId; - this.firstTokenIndex = firstTokenIndex; + this._firstTokenIndex = firstTokenIndex; this._lastTokenIndex = lastTokenIndex; this.firstCharOffset = firstCharOffset; this._lastCharOffset = lastCharOffset; @@ -67,28 +67,16 @@ export class ScopedLineTokens { return actualLineContent.substring(0, this.firstCharOffset + offset); } - public getCount(): number { - return this._lastTokenIndex - this.firstTokenIndex; + public getTokenCount(): number { + return this._lastTokenIndex - this._firstTokenIndex; } public findTokenIndexAtOffset(offset: number): number { - return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this.firstTokenIndex; + return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this._firstTokenIndex; } public getStandardTokenType(tokenIndex: number): StandardTokenType { - return this._actual.getStandardTokenType(tokenIndex + this.firstTokenIndex); - } - - public getStartOffset(tokenIndex: number): number { - return this._actual.getStartOffset(tokenIndex + this.firstTokenIndex) - this.firstCharOffset; - } - - public getEndOffset(tokenIndex: number): number { - return this._actual.getEndOffset(tokenIndex + this.firstTokenIndex) - this.firstCharOffset; - } - - public getLanguageId(tokenIndex: number): string { - return this._actual.getLanguageId(tokenIndex + this.firstTokenIndex); + return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } } diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index dce6bc5d07f72..681e26088acc4 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -9,7 +9,7 @@ import { ITextModel } from 'vs/editor/common/model'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; @@ -20,9 +20,9 @@ import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; */ export class ProcessedIndentRulesSupport { - private readonly _indentationLineProcessor: IndentationLineProcessor; - private readonly _indentRulesSupport: IndentRulesSupport; private readonly _model: IVirtualModel; + private readonly _indentRulesSupport: IndentRulesSupport; + private readonly _indentationLineProcessor: IndentationLineProcessor; constructor( model: IVirtualModel, @@ -128,19 +128,9 @@ export class IndentationContextProcessor { const scopedLineLength = scopedLineContent.length; const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineLength + 1; - const firstTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(columnIndexWithinScope); - const lastTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(scopedLineLength - 1) + 1; const line = scopedLineContent.substring(columnIndexWithinScope); - const languageId = scopedLineTokens.languageId; - const processedTokens = new ScopedLineTokens( - lineTokens, - languageId, - firstTokenIndex, - lastTokenIndex, - firstCharacterOffset, - lastCharacterOffset - ); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, processedTokens); + const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, slicedTokens); return processedLine; } @@ -149,20 +139,10 @@ export class IndentationContextProcessor { const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const firstTokenIndex = scopedLineTokens.firstTokenIndex; - const lastTokenIndex = scopedLineTokens.firstTokenIndex + scopedLineTokens.findTokenIndexAtOffset(columnIndexWithinScope) + 1; - const languageId = scopedLineTokens.languageId; - const processedTokens = new ScopedLineTokens( - lineTokens, - languageId, - firstTokenIndex, - lastTokenIndex, - firstCharacterOffset, - lastCharacterOffset - ); const scopedLineContent = scopedLineTokens.getLineContent(); const line = scopedLineContent.substring(0, columnIndexWithinScope); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, processedTokens); + const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, slicedTokens); return processedLine; } @@ -178,7 +158,10 @@ export class IndentationContextProcessor { if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { // The line above ends with text belonging to the same mode const previousLine = previousLineScopedLineTokens.getLineContent(); - processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousLineScopedLineTokens); + const firstCharacterOffset = previousLineScopedLineTokens.firstCharOffset; + const lastCharacterOffset = firstCharacterOffset + previousLine.length; + const previousSlicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousSlicedTokens); } } return processedPreviousLine; @@ -203,7 +186,7 @@ class IndentationLineProcessor { return processedLine; } - getProcessedLineForLineAndTokens(line: string, tokens: LineTokens | ScopedLineTokens): string { + getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { // Utility functions const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, processedLine: string): { processedCharacterOffset: number, processedLine: string } => { diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index da11aa7aa297f..a2d0a317635e2 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -10,6 +10,7 @@ export interface IViewLineTokens { equals(other: IViewLineTokens): boolean; getCount(): number; getForeground(tokenIndex: number): ColorId; + getStartOffset(tokenIndex: number): number; getEndOffset(tokenIndex: number): number; getClassName(tokenIndex: number): string; getInlineStyle(tokenIndex: number, colorMap: string[]): string; @@ -18,6 +19,7 @@ export interface IViewLineTokens { getLineContent(): string; getMetadata(tokenIndex: number): number; getLanguageId(tokenIndex: number): string; + getStandardTokenType(tokenIndex: number): StandardTokenType; } export class LineTokens implements IViewLineTokens { @@ -297,6 +299,11 @@ class SliceLineTokens implements IViewLineTokens { return this._source.getForeground(this._firstTokenIndex + tokenIndex); } + public getStartOffset(tokenIndex: number): number { + const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); + return Math.max(this._startOffset, tokenStartOffset) - this._startOffset + this._deltaOffset; + } + public getEndOffset(tokenIndex: number): number { const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); return Math.min(this._endOffset, tokenEndOffset) - this._startOffset + this._deltaOffset; @@ -317,4 +324,8 @@ class SliceLineTokens implements IViewLineTokens { public findTokenIndexAtOffset(offset: number): number { return this._source.findTokenIndexAtOffset(offset + this._startOffset - this._deltaOffset) - this._firstTokenIndex; } + + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._source.getStandardTokenType(this._firstTokenIndex + tokenIndex); + } } diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index 8217c2042fb02..cf7e781531df4 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { ColorId, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; +import { ColorId, TokenMetadata, ITokenPresentation, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; /** * A token on a line. @@ -38,6 +38,10 @@ export class TestLineToken { return TokenMetadata.getPresentationFromMetadata(this._metadata); } + public getStandardTokenType(): StandardTokenType { + return TokenMetadata.getTokenType(this._metadata) + } + private static _equals(a: TestLineToken, b: TestLineToken): boolean { return ( a.endIndex === b.endIndex @@ -83,6 +87,13 @@ export class TestLineTokens implements IViewLineTokens { return this._actual[tokenIndex].getForeground(); } + public getStartOffset(tokenIndex: number): number { + if (tokenIndex > 0) { + return this._actual[tokenIndex - 1].endIndex + 1; + } + return 0; + } + public getEndOffset(tokenIndex: number): number { return this._actual[tokenIndex].endIndex; } @@ -114,6 +125,10 @@ export class TestLineTokens implements IViewLineTokens { public getLanguageId(tokenIndex: number): string { throw new Error('Method not implemented.'); } + + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._actual[tokenIndex].getStandardTokenType(); + } } export class TestLineTokenFactory { From 4c178cf470f8212778a9a8735cb8b3cd62320c49 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 12:56:46 +0200 Subject: [PATCH 10/47] polishing the code --- src/vs/editor/common/languages/autoIndent.ts | 8 ++-- .../supports/indentationLineProcessor.ts | 15 +++---- .../codeEditor/test/node/autoindent.test.ts | 44 ++++++++++--------- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index c64b254d761f2..d76bb486c52d9 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -37,7 +37,7 @@ export interface IIndentConverter { * 0: every line above are invalid * else: nearest preceding line of the same language */ -function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentRulesSupport: ProcessedIndentRulesSupport) { +function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, processedIndentRulesSupport: ProcessedIndentRulesSupport) { const languageId = model.tokenization.getLanguageIdAtPosition(lineNumber, 0); if (lineNumber > 1) { let lastLineNumber: number; @@ -48,7 +48,7 @@ function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentR return resultLineNumber; } const text = model.getLineContent(lastLineNumber); - if (indentRulesSupport.shouldIgnore(lastLineNumber) || /^\s+$/.test(text) || text === '') { + if (processedIndentRulesSupport.shouldIgnore(lastLineNumber) || /^\s+$/.test(text) || text === '') { resultLineNumber = lastLineNumber; continue; } @@ -237,6 +237,7 @@ export function getGoodIndentForLine( if (!indentRulesSupport) { return null; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageConfigurationService); const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); @@ -345,7 +346,8 @@ export function getIndentForEnter( }; const embeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); - const currentLineIndent = strings.getLeadingWhitespace(model.getLineContent(range.startLineNumber)); + const currentLine = model.getLineContent(range.startLineNumber); + const currentLineIndent = strings.getLeadingWhitespace(currentLine); const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); if (!afterEnterAction) { const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 681e26088acc4..52becc6dea764 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -125,12 +125,11 @@ export class IndentationContextProcessor { lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); } const scopedLineContent = scopedLineTokens.getLineContent(); - const scopedLineLength = scopedLineContent.length; const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; - const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineLength + 1; - const line = scopedLineContent.substring(columnIndexWithinScope); + const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length + 1; + const slicedLine = scopedLineContent.substring(columnIndexWithinScope); const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, slicedTokens); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); return processedLine; } @@ -140,9 +139,9 @@ export class IndentationContextProcessor { const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const scopedLineContent = scopedLineTokens.getLineContent(); - const line = scopedLineContent.substring(0, columnIndexWithinScope); + const slicedLine = scopedLineContent.substring(0, columnIndexWithinScope); const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(line, slicedTokens); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); return processedLine; } @@ -160,8 +159,8 @@ export class IndentationContextProcessor { const previousLine = previousLineScopedLineTokens.getLineContent(); const firstCharacterOffset = previousLineScopedLineTokens.firstCharOffset; const lastCharacterOffset = firstCharacterOffset + previousLine.length; - const previousSlicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousSlicedTokens); + const previousLineTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousLineTokens); } } return processedPreviousLine; diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 2b77dcc17b5e5..01f1622fe6e12 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -55,22 +55,27 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo return languageConfigurationService.register(languageId, languageConfig); } -function registerTokenizationSupport(languageService: ILanguageService, tokens: { startIndex: number; value: number }[][], languageId: LanguageId): IDisposable { +interface TokenData { + endIndex: number; + metadata: number +} + +function registerTokenizationSupport(languageService: ILanguageService, tokens: TokenData[][], languageId: LanguageId): IDisposable { let lineIndex = 0; + const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); const tokenizationSupport: ITokenizationSupport = { getInitialState: () => NullState, tokenize: undefined!, tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { const tokensOnLine = tokens[lineIndex++]; - const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); - const result = new Uint32Array(2 * tokensOnLine.length); + const tokensArray = new Uint32Array(2 * tokensOnLine.length); for (let i = 0; i < tokensOnLine.length; i++) { - result[2 * i] = tokensOnLine[i].startIndex; - result[2 * i + 1] = + tokensArray[2 * i] = tokensOnLine[i].endIndex; + tokensArray[2 * i + 1] = ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); + | (tokensOnLine[i].metadata << MetadataConsts.TOKEN_TYPE_OFFSET)); } - return new EncodedTokenizationResult(result, state); + return new EncodedTokenizationResult(tokensArray, state); } }; return TokenizationRegistry.register(languageId, tokenizationSupport); @@ -193,14 +198,14 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const foo = `{`;', ' ', ].join('\n'); - const tokens = [ + const tokens: TokenData[][] = [ [ - { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, - { startIndex: 9, value: 0 }, { startIndex: 10, value: 0 }, { startIndex: 11, value: 0 }, - { startIndex: 12, value: 2 }, { startIndex: 13, value: 2 }, { startIndex: 14, value: 2 }, - { startIndex: 15, value: 0 }, { startIndex: 16, value: 0 }, + { endIndex: 5, metadata: 0 }, { endIndex: 6, metadata: 0 }, { endIndex: 9, metadata: 0 }, + { endIndex: 10, metadata: 0 }, { endIndex: 11, metadata: 0 }, { endIndex: 12, metadata: 2 }, + { endIndex: 13, metadata: 2 }, { endIndex: 14, metadata: 2 }, { endIndex: 15, metadata: 0 }, + { endIndex: 16, metadata: 0 }, ], - [{ startIndex: 0, value: 0 }, { startIndex: 1, value: 0 }] + [{ endIndex: 4, metadata: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); @@ -304,20 +309,19 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const r = /{/;', ' ', ].join('\n'); - const tokens = [ + const tokens: TokenData[][] = [ [ - { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, - { startIndex: 7, value: 0 }, { startIndex: 8, value: 0 }, { startIndex: 9, value: 3 }, - { startIndex: 10, value: 3 }, { startIndex: 11, value: 3 }, { startIndex: 12, value: 3 }, - { startIndex: 13, value: 0 }, { startIndex: 14, value: 0 } + { endIndex: 5, metadata: 0 }, { endIndex: 6, metadata: 0 }, { endIndex: 7, metadata: 0 }, + { endIndex: 8, metadata: 0 }, { endIndex: 9, metadata: 3 }, { endIndex: 10, metadata: 3 }, + { endIndex: 11, metadata: 3 }, { endIndex: 12, metadata: 3 }, { endIndex: 13, metadata: 0 }, + { endIndex: 14, metadata: 0 } ], - [{ startIndex: 0, value: 0 }, { startIndex: 1, value: 0 }] + [{ endIndex: 4, metadata: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; From 958a7609e42fc2b4eae229ed78672ae6e25c37ef Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 14:58:27 +0200 Subject: [PATCH 11/47] using start indices instead --- .../codeEditor/test/node/autoindent.test.ts | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 01f1622fe6e12..59967c923fbf7 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -56,7 +56,7 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo } interface TokenData { - endIndex: number; + startIndex: number; metadata: number } @@ -70,7 +70,7 @@ function registerTokenizationSupport(languageService: ILanguageService, tokens: const tokensOnLine = tokens[lineIndex++]; const tokensArray = new Uint32Array(2 * tokensOnLine.length); for (let i = 0; i < tokensOnLine.length; i++) { - tokensArray[2 * i] = tokensOnLine[i].endIndex; + tokensArray[2 * i] = tokensOnLine[i].startIndex; tokensArray[2 * i + 1] = ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) | (tokensOnLine[i].metadata << MetadataConsts.TOKEN_TYPE_OFFSET)); @@ -200,18 +200,17 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ].join('\n'); const tokens: TokenData[][] = [ [ - { endIndex: 5, metadata: 0 }, { endIndex: 6, metadata: 0 }, { endIndex: 9, metadata: 0 }, - { endIndex: 10, metadata: 0 }, { endIndex: 11, metadata: 0 }, { endIndex: 12, metadata: 2 }, - { endIndex: 13, metadata: 2 }, { endIndex: 14, metadata: 2 }, { endIndex: 15, metadata: 0 }, - { endIndex: 16, metadata: 0 }, + { startIndex: 0, metadata: 0 }, { startIndex: 5, metadata: 0 }, { startIndex: 6, metadata: 0 }, + { startIndex: 9, metadata: 0 }, { startIndex: 10, metadata: 0 }, { startIndex: 11, metadata: 0 }, + { startIndex: 12, metadata: 2 }, { startIndex: 13, metadata: 2 }, { startIndex: 14, metadata: 2 }, + { startIndex: 15, metadata: 0 }, { startIndex: 16, metadata: 0 } ], - [{ endIndex: 4, metadata: 0 }] + [{ startIndex: 0, metadata: 0 }, { startIndex: 4, metadata: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; @@ -311,12 +310,12 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ].join('\n'); const tokens: TokenData[][] = [ [ - { endIndex: 5, metadata: 0 }, { endIndex: 6, metadata: 0 }, { endIndex: 7, metadata: 0 }, - { endIndex: 8, metadata: 0 }, { endIndex: 9, metadata: 3 }, { endIndex: 10, metadata: 3 }, - { endIndex: 11, metadata: 3 }, { endIndex: 12, metadata: 3 }, { endIndex: 13, metadata: 0 }, - { endIndex: 14, metadata: 0 } + { startIndex: 0, metadata: 0 }, { startIndex: 5, metadata: 0 }, { startIndex: 6, metadata: 0 }, + { startIndex: 7, metadata: 0 }, { startIndex: 8, metadata: 0 }, { startIndex: 9, metadata: 3 }, + { startIndex: 10, metadata: 3 }, { startIndex: 11, metadata: 3 }, { startIndex: 12, metadata: 3 }, + { startIndex: 13, metadata: 0 }, { startIndex: 14, metadata: 0 } ], - [{ endIndex: 4, metadata: 0 }] + [{ startIndex: 0, metadata: 0 }, { startIndex: 4, metadata: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); From 50fc18339e88d88401ead33bf8539b1da860330e Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:03:28 +0200 Subject: [PATCH 12/47] using value everywhere --- .../test/browser/indentation.test.ts | 9 +++++-- .../codeEditor/test/node/autoindent.test.ts | 24 +++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index d66f65df07524..12c10abe1e065 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -11,7 +11,7 @@ import { createTextModel } from 'vs/editor/test/common/testTextModel'; import { TestInstantiationService } from 'vs/platform/instantiation/test/common/instantiationServiceMock'; import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; -import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; +import { MetadataConsts, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { NullState } from 'vs/editor/common/languages/nullTokenize'; @@ -118,7 +118,12 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -function registerTokens(instantiationService: TestInstantiationService, tokens: { startIndex: number; value: number }[][], languageId: string, disposables: DisposableStore) { +interface TokenData { + startIndex: number; + value: StandardTokenType +} + +function registerTokens(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string, disposables: DisposableStore) { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); const tokenizationSupport: ITokenizationSupport = { diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 59967c923fbf7..8f4f35a83098d 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -57,7 +57,7 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo interface TokenData { startIndex: number; - metadata: number + value: number } function registerTokenizationSupport(languageService: ILanguageService, tokens: TokenData[][], languageId: LanguageId): IDisposable { @@ -73,7 +73,7 @@ function registerTokenizationSupport(languageService: ILanguageService, tokens: tokensArray[2 * i] = tokensOnLine[i].startIndex; tokensArray[2 * i + 1] = ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (tokensOnLine[i].metadata << MetadataConsts.TOKEN_TYPE_OFFSET)); + | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); } return new EncodedTokenizationResult(tokensArray, state); } @@ -200,12 +200,12 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ].join('\n'); const tokens: TokenData[][] = [ [ - { startIndex: 0, metadata: 0 }, { startIndex: 5, metadata: 0 }, { startIndex: 6, metadata: 0 }, - { startIndex: 9, metadata: 0 }, { startIndex: 10, metadata: 0 }, { startIndex: 11, metadata: 0 }, - { startIndex: 12, metadata: 2 }, { startIndex: 13, metadata: 2 }, { startIndex: 14, metadata: 2 }, - { startIndex: 15, metadata: 0 }, { startIndex: 16, metadata: 0 } + { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, + { startIndex: 9, value: 0 }, { startIndex: 10, value: 0 }, { startIndex: 11, value: 0 }, + { startIndex: 12, value: 2 }, { startIndex: 13, value: 2 }, { startIndex: 14, value: 2 }, + { startIndex: 15, value: 0 }, { startIndex: 16, value: 0 } ], - [{ startIndex: 0, metadata: 0 }, { startIndex: 4, metadata: 0 }] + [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); @@ -310,12 +310,12 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ].join('\n'); const tokens: TokenData[][] = [ [ - { startIndex: 0, metadata: 0 }, { startIndex: 5, metadata: 0 }, { startIndex: 6, metadata: 0 }, - { startIndex: 7, metadata: 0 }, { startIndex: 8, metadata: 0 }, { startIndex: 9, metadata: 3 }, - { startIndex: 10, metadata: 3 }, { startIndex: 11, metadata: 3 }, { startIndex: 12, metadata: 3 }, - { startIndex: 13, metadata: 0 }, { startIndex: 14, metadata: 0 } + { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, + { startIndex: 7, value: 0 }, { startIndex: 8, value: 0 }, { startIndex: 9, value: 3 }, + { startIndex: 10, value: 3 }, { startIndex: 11, value: 3 }, { startIndex: 12, value: 3 }, + { startIndex: 13, value: 0 }, { startIndex: 14, value: 0 } ], - [{ startIndex: 0, metadata: 0 }, { startIndex: 4, metadata: 0 }] + [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] ]; disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); From 8b350c58f4072c20e337193ad3059d9514e47072 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:04:20 +0200 Subject: [PATCH 13/47] using the token data to type the tokens --- .../indentation/test/browser/indentation.test.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 12c10abe1e065..8a6f78dc8de0f 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -420,7 +420,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { ' */', 'function a() {}' ].join('\n'); - const tokens = [ + const tokens: TokenData[][] = [ [ { startIndex: 0, value: 1 }, { startIndex: 3, value: 1 }, @@ -580,7 +580,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { ' * }', ' */' ].join('\n'); - const tokens = [ + const tokens: TokenData[][] = [ [ { startIndex: 0, value: 1 }, { startIndex: 3, value: 1 }, @@ -732,7 +732,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { disposables.add(model); withTestCodeEditor(model, { autoIndent: 'full' }, (editor, viewModel, instantiationService) => { - const tokens = [ + const tokens: TokenData[][] = [ [{ startIndex: 0, value: 0 }, { startIndex: 8, value: 0 }, { startIndex: 9, value: 0 }, { startIndex: 12, value: 0 }, { startIndex: 13, value: 0 }, { startIndex: 14, value: 0 }, { startIndex: 15, value: 0 }, { startIndex: 16, value: 0 }], [{ startIndex: 0, value: 1 }, { startIndex: 2, value: 1 }, { startIndex: 3, value: 1 }, { startIndex: 10, value: 1 }], [{ startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, { startIndex: 9, value: 0 }, { startIndex: 10, value: 0 }, { startIndex: 11, value: 0 }, { startIndex: 12, value: 0 }, { startIndex: 14, value: 0 }], @@ -1003,7 +1003,7 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { disposables.add(model); withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { - const tokens = [ + const tokens: TokenData[][] = [ [{ startIndex: 0, value: 1 }], [{ startIndex: 0, value: 1 }], [{ startIndex: 0, value: 1 }] @@ -1399,7 +1399,7 @@ suite('Auto Indent On Type - PHP', () => { disposables.add(model); withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { - const tokens = [[{ startIndex: 0, value: 0 }, { startIndex: 13, value: 2 }, { startIndex: 16, value: 0 }]]; + const tokens: TokenData[][] = [[{ startIndex: 0, value: 0 }, { startIndex: 13, value: 2 }, { startIndex: 16, value: 0 }]]; registerTokens(instantiationService, tokens, languageId, disposables); registerLanguage(instantiationService, languageId, Language.PHP, disposables); editor.setSelection(new Selection(1, 19, 1, 19)); From f43cb1afadf4cd6aa7f773f241a1685118323ec3 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:05:34 +0200 Subject: [PATCH 14/47] setting to number instead of standard token type --- .../contrib/indentation/test/browser/indentation.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 8a6f78dc8de0f..8892975bc6606 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -11,7 +11,7 @@ import { createTextModel } from 'vs/editor/test/common/testTextModel'; import { TestInstantiationService } from 'vs/platform/instantiation/test/common/instantiationServiceMock'; import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; -import { MetadataConsts, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { NullState } from 'vs/editor/common/languages/nullTokenize'; @@ -120,7 +120,7 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe interface TokenData { startIndex: number; - value: StandardTokenType + value: number; } function registerTokens(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string, disposables: DisposableStore) { From 8f3a9eee31aeec6a7f0237729d6a55817ce89064 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:09:39 +0200 Subject: [PATCH 15/47] using token data from autoindenttest.ts --- .../contrib/indentation/test/browser/indentation.test.ts | 6 +----- .../contrib/codeEditor/test/node/autoindent.test.ts | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 8892975bc6606..ddd5edb814dec 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -23,6 +23,7 @@ import { cppOnEnterRules, htmlOnEnterRules, javascriptOnEnterRules, phpOnEnterRu import { TypeOperations } from 'vs/editor/common/cursor/cursorTypeOperations'; import { cppBracketRules, goBracketRules, htmlBracketRules, latexBracketRules, luaBracketRules, phpBracketRules, rubyBracketRules, typescriptBracketRules, vbBracketRules } from 'vs/editor/test/common/modes/supports/bracketRules'; import { latexAutoClosingPairsRules } from 'vs/editor/test/common/modes/supports/autoClosingPairsRules'; +import { TokenData } from 'vs/workbench/contrib/codeEditor/test/node/autoindent.test'; enum Language { TypeScript, @@ -118,11 +119,6 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -interface TokenData { - startIndex: number; - value: number; -} - function registerTokens(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string, disposables: DisposableStore) { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 8f4f35a83098d..e2fa20cfd9c80 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -55,7 +55,7 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo return languageConfigurationService.register(languageId, languageConfig); } -interface TokenData { +export interface TokenData { startIndex: number; value: number } From 7f4ac61bcf6f2ffe37992d25365c587c0c2cdb8f Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:16:39 +0200 Subject: [PATCH 16/47] using same code in both test files --- .../test/browser/indentation.test.ts | 22 ++++++----- .../codeEditor/test/node/autoindent.test.ts | 38 ++----------------- 2 files changed, 17 insertions(+), 43 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index ddd5edb814dec..150bbc00cca5b 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; -import { DisposableStore } from 'vs/base/common/lifecycle'; +import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createTextModel } from 'vs/editor/test/common/testTextModel'; @@ -23,7 +23,6 @@ import { cppOnEnterRules, htmlOnEnterRules, javascriptOnEnterRules, phpOnEnterRu import { TypeOperations } from 'vs/editor/common/cursor/cursorTypeOperations'; import { cppBracketRules, goBracketRules, htmlBracketRules, latexBracketRules, luaBracketRules, phpBracketRules, rubyBracketRules, typescriptBracketRules, vbBracketRules } from 'vs/editor/test/common/modes/supports/bracketRules'; import { latexAutoClosingPairsRules } from 'vs/editor/test/common/modes/supports/autoClosingPairsRules'; -import { TokenData } from 'vs/workbench/contrib/codeEditor/test/node/autoindent.test'; enum Language { TypeScript, @@ -119,7 +118,12 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -function registerTokens(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string, disposables: DisposableStore) { +export interface TokenData { + startIndex: number; + value: number +} + +export function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string): IDisposable { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); const tokenizationSupport: ITokenizationSupport = { @@ -140,7 +144,7 @@ function registerTokens(instantiationService: TestInstantiationService, tokens: return new EncodedTokenizationResult(result, state); } }; - disposables.add(TokenizationRegistry.register(languageId, tokenizationSupport)); + return TokenizationRegistry.register(languageId, tokenizationSupport); } suite('Change Indentation to Spaces - TypeScript/Javascript', () => { @@ -444,7 +448,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { ] ]; registerLanguage(instantiationService, languageId, Language.TypeScript, disposables); - registerTokens(instantiationService, tokens, languageId, disposables); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const autoIndentOnPasteController = editor.registerAndInstantiateContribution(AutoIndentOnPaste.ID, AutoIndentOnPaste); viewModel.paste(pasteText, true, undefined, 'keyboard'); autoIndentOnPasteController.trigger(new Range(1, 1, 4, 16)); @@ -602,7 +606,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { ] ]; registerLanguage(instantiationService, languageId, Language.TypeScript, disposables); - registerTokens(instantiationService, tokens, languageId, disposables); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const autoIndentOnPasteController = editor.registerAndInstantiateContribution(AutoIndentOnPaste.ID, AutoIndentOnPaste); viewModel.paste(text, true, undefined, 'keyboard'); autoIndentOnPasteController.trigger(new Range(1, 1, 4, 4)); @@ -735,7 +739,7 @@ suite('Auto Indent On Paste - TypeScript/JavaScript', () => { [{ startIndex: 0, value: 0 }, { startIndex: 1, value: 0 }] ]; registerLanguage(instantiationService, languageId, Language.TypeScript, disposables); - registerTokens(instantiationService, tokens, languageId, disposables); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(2, 1, 2, 1)); const text = [ @@ -1005,7 +1009,7 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { [{ startIndex: 0, value: 1 }] ]; registerLanguage(instantiationService, languageId, Language.TypeScript, disposables); - registerTokens(instantiationService, tokens, languageId, disposables); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(2, 23, 2, 23)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ @@ -1396,7 +1400,7 @@ suite('Auto Indent On Type - PHP', () => { withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { const tokens: TokenData[][] = [[{ startIndex: 0, value: 0 }, { startIndex: 13, value: 2 }, { startIndex: 16, value: 0 }]]; - registerTokens(instantiationService, tokens, languageId, disposables); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); registerLanguage(instantiationService, languageId, Language.PHP, disposables); editor.setSelection(new Selection(1, 19, 1, 19)); viewModel.type("\n", 'keyboard'); diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index e2fa20cfd9c80..7426dd9a12135 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -19,9 +19,7 @@ import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { trimTrailingWhitespace } from 'vs/editor/common/commands/trimTrailingWhitespaceCommand'; import { execSync } from 'child_process'; import { ILanguageService } from 'vs/editor/common/languages/language'; -import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; -import { NullState } from 'vs/editor/common/languages/nullTokenize'; -import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; +import { registerTokenizationSupport, TokenData } from 'vs/editor/contrib/indentation/test/browser/indentation.test'; function getIRange(range: IRange): IRange { return { @@ -55,32 +53,6 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo return languageConfigurationService.register(languageId, languageConfig); } -export interface TokenData { - startIndex: number; - value: number -} - -function registerTokenizationSupport(languageService: ILanguageService, tokens: TokenData[][], languageId: LanguageId): IDisposable { - let lineIndex = 0; - const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); - const tokenizationSupport: ITokenizationSupport = { - getInitialState: () => NullState, - tokenize: undefined!, - tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { - const tokensOnLine = tokens[lineIndex++]; - const tokensArray = new Uint32Array(2 * tokensOnLine.length); - for (let i = 0; i < tokensOnLine.length; i++) { - tokensArray[2 * i] = tokensOnLine[i].startIndex; - tokensArray[2 * i + 1] = - ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); - } - return new EncodedTokenizationResult(tokensArray, state); - } - }; - return TokenizationRegistry.register(languageId, tokenizationSupport); -} - suite('Auto-Reindentation - TypeScript/JavaScript', () => { const languageId = LanguageId.TypeScript; @@ -88,14 +60,12 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { let disposables: DisposableStore; let instantiationService: TestInstantiationService; let languageConfigurationService: ILanguageConfigurationService; - let languageService: ILanguageService; setup(() => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); disposables.add(instantiationService); - languageService = instantiationService.get(ILanguageService); - disposables.add(registerLanguage(languageService, languageId)); + disposables.add(registerLanguage(instantiationService.get(ILanguageService), languageId)); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); }); @@ -207,7 +177,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ], [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] ]; - disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); @@ -317,7 +287,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ], [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] ]; - disposables.add(registerTokenizationSupport(languageService, tokens, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); From f871d5bab6b7a46995e54e2590c8c59702da1687 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 15:31:59 +0200 Subject: [PATCH 17/47] placing instantiation service into the registerLanguage method --- .../contrib/codeEditor/test/node/autoindent.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 7426dd9a12135..45be9e0922bb4 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -34,7 +34,8 @@ const enum LanguageId { TypeScript = 'ts-test' } -function registerLanguage(languageService: ILanguageService, languageId: LanguageId): IDisposable { +function registerLanguage(instantiationService: TestInstantiationService, languageId: LanguageId): IDisposable { + const languageService = instantiationService.get(ILanguageService) return languageService.registerLanguage({ id: languageId }); } @@ -65,7 +66,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); disposables.add(instantiationService); - disposables.add(registerLanguage(instantiationService.get(ILanguageService), languageId)); + disposables.add(registerLanguage(instantiationService, languageId)); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); }); From 1113593476bc6a1b2f27147d01d6bfa285dbb8a7 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Fri, 19 Apr 2024 16:24:26 +0200 Subject: [PATCH 18/47] copying object into the node js autoindent.ts --- .../test/browser/indentation.test.ts | 4 +-- .../codeEditor/test/node/autoindent.test.ts | 33 ++++++++++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 150bbc00cca5b..b43f63b62995d 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -118,12 +118,12 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -export interface TokenData { +interface TokenData { startIndex: number; value: number } -export function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string): IDisposable { +function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string): IDisposable { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); const tokenizationSupport: ITokenizationSupport = { diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 45be9e0922bb4..453d4095f70ed 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -19,7 +19,9 @@ import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { trimTrailingWhitespace } from 'vs/editor/common/commands/trimTrailingWhitespaceCommand'; import { execSync } from 'child_process'; import { ILanguageService } from 'vs/editor/common/languages/language'; -import { registerTokenizationSupport, TokenData } from 'vs/editor/contrib/indentation/test/browser/indentation.test'; +import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; +import { NullState } from 'vs/editor/common/languages/nullTokenize'; +import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; function getIRange(range: IRange): IRange { return { @@ -54,6 +56,35 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo return languageConfigurationService.register(languageId, languageConfig); } +interface TokenData { + startIndex: number; + value: number +} + +function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string): IDisposable { + let lineIndex = 0; + const languageService = instantiationService.get(ILanguageService); + const tokenizationSupport: ITokenizationSupport = { + getInitialState: () => NullState, + tokenize: undefined!, + tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { + const tokensOnLine = tokens[lineIndex++]; + const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); + const result = new Uint32Array(2 * tokensOnLine.length); + for (let i = 0; i < tokensOnLine.length; i++) { + result[2 * i] = tokensOnLine[i].startIndex; + result[2 * i + 1] = + ( + (encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET) + ); + } + return new EncodedTokenizationResult(result, state); + } + }; + return TokenizationRegistry.register(languageId, tokenizationSupport); +} + suite('Auto-Reindentation - TypeScript/JavaScript', () => { const languageId = LanguageId.TypeScript; From 410428f4db92f93a8638e1dabf76f4edaa0700ce Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 22 Apr 2024 14:50:28 +0200 Subject: [PATCH 19/47] combining methods into one method --- .../supports/indentationLineProcessor.ts | 87 +++++++++---------- .../contrib/indentation/common/indentation.ts | 6 +- 2 files changed, 43 insertions(+), 50 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 52becc6dea764..adf490c79e1b4 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -20,7 +20,6 @@ import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; */ export class ProcessedIndentRulesSupport { - private readonly _model: IVirtualModel; private readonly _indentRulesSupport: IndentRulesSupport; private readonly _indentationLineProcessor: IndentationLineProcessor; @@ -29,54 +28,38 @@ export class ProcessedIndentRulesSupport { indentRulesSupport: IndentRulesSupport, languageConfigurationService: ILanguageConfigurationService ) { - this._model = model; this._indentRulesSupport = indentRulesSupport; this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } - public shouldIncrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); + public shouldIncrease(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIncrease(processedLine); } - public shouldDecrease(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); + public shouldDecrease(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldDecrease(processedLine); } - public shouldIgnore(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); + public shouldIgnore(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIgnore(processedLine); } - public shouldIndentNextLine(lineNumber: number): boolean { - const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); + public shouldIndentNextLine(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIndentNextLine(processedLine); } - public shouldIncreaseAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { - const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); - return this._indentRulesSupport.shouldIncrease(processedLine); - } - - public shouldIndentNextLineAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { - const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); - return this._indentRulesSupport.shouldIndentNextLine(processedLine); - } - - public shouldDecreaseAfterSettingIndentation(lineNumber: number, newIndentation: string): boolean { - const processedLine = this._adjustIndentationForLineAndProcess(lineNumber, newIndentation); - return this._indentRulesSupport.shouldDecrease(processedLine); - } - - private _adjustIndentationForLineAndProcess(lineNumber: number, newIndentation: string): string { - const currentLine = this._model.getLineContent(lineNumber); - const currentIndentation = strings.getLeadingWhitespace(currentLine); - const currentTokens = this._model.tokenization.getLineTokens(lineNumber); - const indentationDifference = newIndentation.length - currentIndentation.length; - const newLine = newIndentation + currentLine.substring(currentIndentation.length);; - const newTokens = currentTokens.shiftTokenOffsetsBy(indentationDifference, newLine); - return this._indentationLineProcessor.getProcessedLineForLineAndTokens(newLine, newTokens); + private _processLine(lineNumber: number, newIndentation?: string) { + let processedLine: string; + if (newIndentation === undefined) { + processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); + } else { + processedLine = this._indentationLineProcessor.getProcessedLineWithIndentation(lineNumber, newIndentation); + } + return processedLine; } } @@ -114,14 +97,26 @@ export class IndentationContextProcessor { return { beforeRangeText, afterRangeText, previousLineText }; } + private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): string { + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; + const firstCharacterOffset = scopedLineTokens.firstCharOffset; + const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; + const scopedLineContent = scopedLineTokens.getLineContent(); + const slicedLine = scopedLineContent.substring(0, columnIndexWithinScope); + const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); + return processedLine; + } + private _getProcessedTextAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): string { let columnIndexWithinScope: number; let lineTokens: LineTokens; if (range.isEmpty()) { - columnIndexWithinScope = range.startColumn - 1 - scopedLineTokens.firstCharOffset; + columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); } else { - columnIndexWithinScope = range.endColumn - 1 - scopedLineTokens.firstCharOffset; + columnIndexWithinScope = (range.endColumn - 1) - scopedLineTokens.firstCharOffset; lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); } const scopedLineContent = scopedLineTokens.getLineContent(); @@ -133,18 +128,6 @@ export class IndentationContextProcessor { return processedLine; } - private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): string { - const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; - const firstCharacterOffset = scopedLineTokens.firstCharOffset; - const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const scopedLineContent = scopedLineTokens.getLineContent(); - const slicedLine = scopedLineContent.substring(0, columnIndexWithinScope); - const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); - return processedLine; - } - private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens) { let processedPreviousLine = ''; if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { @@ -185,6 +168,16 @@ class IndentationLineProcessor { return processedLine; } + getProcessedLineWithIndentation(lineNumber: number, newIndentation: string): string { + const currentLine = this.model.getLineContent(lineNumber); + const currentIndentation = strings.getLeadingWhitespace(currentLine); + const currentTokens = this.model.tokenization.getLineTokens(lineNumber); + const indentationDifference = newIndentation.length - currentIndentation.length; + const newLine = newIndentation + currentLine.substring(currentIndentation.length);; + const newTokens = currentTokens.shiftTokenOffsetsBy(indentationDifference, newLine); + return this.getProcessedLineForLineAndTokens(newLine, newTokens); + } + getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { // Utility functions diff --git a/src/vs/editor/contrib/indentation/common/indentation.ts b/src/vs/editor/contrib/indentation/common/indentation.ts index 32fb40a5f613c..41cc7cb8068b8 100644 --- a/src/vs/editor/contrib/indentation/common/indentation.ts +++ b/src/vs/editor/contrib/indentation/common/indentation.ts @@ -75,7 +75,7 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati const oldIndentation = strings.getLeadingWhitespace(text); const currentIdealIndent = idealIndentForNextLine; - if (processedIndentRulesSupport.shouldDecreaseAfterSettingIndentation(lineNumber, currentIdealIndent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber, currentIdealIndent)) { idealIndentForNextLine = unshiftIndent(idealIndentForNextLine); globalIndent = unshiftIndent(globalIndent); } @@ -89,10 +89,10 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati // In reindent phase, if the line matches `unIndentedLinePattern` we inherit indentation from above lines // but don't change globalIndent and idealIndentForNextLine. continue; - } else if (processedIndentRulesSupport.shouldIncreaseAfterSettingIndentation(lineNumber, currentIdealIndent)) { + } else if (processedIndentRulesSupport.shouldIncrease(lineNumber, currentIdealIndent)) { globalIndent = shiftIndent(globalIndent); idealIndentForNextLine = globalIndent; - } else if (processedIndentRulesSupport.shouldIndentNextLineAfterSettingIndentation(lineNumber, currentIdealIndent)) { + } else if (processedIndentRulesSupport.shouldIndentNextLine(lineNumber, currentIdealIndent)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); } else { idealIndentForNextLine = globalIndent; From cc4da1cb6473a4ee0f970c03757cd06a4b21de11 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 22 Apr 2024 15:10:59 +0200 Subject: [PATCH 20/47] making fields private, adding js docs --- .../languages/supports/indentationLineProcessor.ts | 13 +++++++++++-- src/vs/editor/common/tokens/lineTokens.ts | 4 ++-- .../contrib/codeEditor/test/node/autoindent.test.ts | 6 ++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index adf490c79e1b4..768ccc42a9dec 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -157,10 +157,13 @@ export class IndentationContextProcessor { class IndentationLineProcessor { constructor( - protected readonly model: IVirtualModel, - protected readonly languageConfigurationService: ILanguageConfigurationService + private readonly model: IVirtualModel, + private readonly languageConfigurationService: ILanguageConfigurationService ) { } + /** + * Get the processed line for the given line number. Remove the language configuration brackets from the regex, string and comment tokens. + */ getProcessedLine(lineNumber: number): string { const lineContent = this.model.getLineContent(lineNumber); const tokens = this.model.tokenization.getLineTokens(lineNumber); @@ -168,6 +171,9 @@ class IndentationLineProcessor { return processedLine; } + /** + * Replace the indentation of the line at the given line number with the new indentation and process the line - remove the language configuration brackets from the regex, string and comment tokens. + */ getProcessedLineWithIndentation(lineNumber: number, newIndentation: string): string { const currentLine = this.model.getLineContent(lineNumber); const currentIndentation = strings.getLeadingWhitespace(currentLine); @@ -178,6 +184,9 @@ class IndentationLineProcessor { return this.getProcessedLineForLineAndTokens(newLine, newTokens); } + /** + * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. + */ getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { // Utility functions diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index a2d0a317635e2..cf74a35bccab8 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -183,13 +183,13 @@ export class LineTokens implements IViewLineTokens { return low; } - public shiftTokenOffsetsBy(delta: number, newText: string): LineTokens { + public shiftTokenOffsetsBy(delta: number, newTokenText: string): LineTokens { const newTokens = new Array(); for (let i = 0; i < this.getCount(); i++) { newTokens.push(this.getEndOffset(i) + delta); newTokens.push(this.getMetadata(i)); } - return new LineTokens(new Uint32Array(newTokens), newText, this._languageIdCodec); + return new LineTokens(new Uint32Array(newTokens), newTokenText, this._languageIdCodec); } /** diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 453d4095f70ed..ff66287513dd8 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -74,10 +74,8 @@ function registerTokenizationSupport(instantiationService: TestInstantiationServ for (let i = 0; i < tokensOnLine.length; i++) { result[2 * i] = tokensOnLine[i].startIndex; result[2 * i + 1] = - ( - (encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET) - ); + ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); } return new EncodedTokenizationResult(result, state); } From 34300995822e270952c5aef059e04932be7f6f21 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 22 Apr 2024 15:26:45 +0200 Subject: [PATCH 21/47] renaming variables, adding js docs --- .../supports/indentationLineProcessor.ts | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 768ccc42a9dec..e3674ff17eb60 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -32,21 +32,33 @@ export class ProcessedIndentRulesSupport { this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } + /** + * Apply the new indentation and return whether the indentation level should be increased after the given line number + */ public shouldIncrease(lineNumber: number, newIndentation?: string): boolean { const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIncrease(processedLine); } + /** + * Apply the new indentation and return whether the indentation level should be decreased after the given line number + */ public shouldDecrease(lineNumber: number, newIndentation?: string): boolean { const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldDecrease(processedLine); } + /** + * Apply the new indentation and return whether the indentation level should remain unchanged at the given line number + */ public shouldIgnore(lineNumber: number, newIndentation?: string): boolean { const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIgnore(processedLine); } + /** + * Apply the new indentation and return whether the indentation level should increase on the line after the given line number + */ public shouldIndentNextLine(lineNumber: number, newIndentation?: string): boolean { const processedLine = this._processLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIndentNextLine(processedLine); @@ -83,6 +95,9 @@ export class IndentationContextProcessor { this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } + /** + * Returns the processed text, stripped from the language configuration brackets within the string, comment and regex tokens, around the given range + */ getProcessedContextAroundRange(range: Range): { beforeRangeText: string; afterRangeText: string; @@ -190,29 +205,29 @@ class IndentationLineProcessor { getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { // Utility functions - const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, characterOffset: number, processedLine: string): { processedCharacterOffset: number, processedLine: string } => { + const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, offset: number, processedLine: string): { processedOffset: number, processedLine: string } => { const result = removeBracketsFromTokenWithIndex(tokenIndex); - const processedCharacterOffset = characterOffset - (result.tokenText.length - result.processedText.length); - const lineBeforeCharacterOffset = processedLine.substring(0, characterOffset + result.tokenStartCharacterOffset); - const lineAfterCharacterOffset = processedLine.substring(characterOffset + result.tokenEndCharacterOffset); - const newProcessedLine = lineBeforeCharacterOffset + result.processedText + lineAfterCharacterOffset; - return { processedCharacterOffset, processedLine: newProcessedLine }; + const processedOffset = offset - (result.tokenText.length - result.processedTokenText.length); + const lineBeforeToken = processedLine.substring(0, offset + result.tokenInitialStartOffset); + const lineAfterToken = processedLine.substring(offset + result.tokenInitialEndOffset); + const newProcessedLine = lineBeforeToken + result.processedTokenText + lineAfterToken; + return { processedOffset, processedLine: newProcessedLine }; }; - const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedText: string; tokenStartCharacterOffset: number; tokenEndCharacterOffset: number } => { - const tokenStartCharacterOffset = tokens.getStartOffset(tokenIndex); - const tokenEndCharacterOffset = tokens.getEndOffset(tokenIndex); - const tokenText = line.substring(tokenStartCharacterOffset, tokenEndCharacterOffset); - const processedText = removeBracketsFromText(tokenText); - return { tokenText, processedText, tokenStartCharacterOffset, tokenEndCharacterOffset }; + const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedTokenText: string; tokenInitialStartOffset: number; tokenInitialEndOffset: number } => { + const tokenInitialStartOffset = tokens.getStartOffset(tokenIndex); + const tokenInitialEndOffset = tokens.getEndOffset(tokenIndex); + const tokenText = line.substring(tokenInitialStartOffset, tokenInitialEndOffset); + const processedTokenText = removeBracketsFromText(tokenText); + return { tokenText, processedTokenText, tokenInitialStartOffset, tokenInitialEndOffset }; } const removeBracketsFromText = (line: string): string => { let processedLine = line; openBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), "g"); + const regex = new RegExp(escapeStringForRegex(bracket), 'g'); processedLine = processedLine.replace(regex, ''); }); closedBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), "g"); + const regex = new RegExp(escapeStringForRegex(bracket), 'g'); processedLine = processedLine.replace(regex, ''); }); return processedLine; @@ -237,7 +252,7 @@ class IndentationLineProcessor { const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - let characterOffset = 0; + let processedOffset = 0; let processedLine = line; for (let i = 0; i < tokens.getCount(); i++) { @@ -246,8 +261,8 @@ class IndentationLineProcessor { || standardTokenType === StandardTokenType.RegEx || standardTokenType === StandardTokenType.Comment ) { - const result = removeBracketsFromTokenWithIndexWithinLine(i, characterOffset, processedLine); - characterOffset = result.processedCharacterOffset; + const result = removeBracketsFromTokenWithIndexWithinLine(i, processedOffset, processedLine); + processedOffset = result.processedOffset; processedLine = result.processedLine; } } From 8b88b2ed4a9ef3caf185334c14948c328ae05d97 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 22 Apr 2024 15:30:17 +0200 Subject: [PATCH 22/47] renaming --- .../languages/supports/indentationLineProcessor.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index e3674ff17eb60..9edcf078cb2f2 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -205,13 +205,13 @@ class IndentationLineProcessor { getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { // Utility functions - const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, offset: number, processedLine: string): { processedOffset: number, processedLine: string } => { + const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, offset: number, line: string): { processedOffset: number, processedLine: string } => { const result = removeBracketsFromTokenWithIndex(tokenIndex); const processedOffset = offset - (result.tokenText.length - result.processedTokenText.length); - const lineBeforeToken = processedLine.substring(0, offset + result.tokenInitialStartOffset); - const lineAfterToken = processedLine.substring(offset + result.tokenInitialEndOffset); - const newProcessedLine = lineBeforeToken + result.processedTokenText + lineAfterToken; - return { processedOffset, processedLine: newProcessedLine }; + const lineBeforeToken = line.substring(0, offset + result.tokenInitialStartOffset); + const lineAfterToken = line.substring(offset + result.tokenInitialEndOffset); + const processedLine = lineBeforeToken + result.processedTokenText + lineAfterToken; + return { processedOffset, processedLine }; }; const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedTokenText: string; tokenInitialStartOffset: number; tokenInitialEndOffset: number } => { const tokenInitialStartOffset = tokens.getStartOffset(tokenIndex); From 97820657cab32062980289604ddc7150426ccddf Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 22 Apr 2024 17:11:09 +0200 Subject: [PATCH 23/47] adding return type to the method `_getProcessedPreviousLine` --- .../common/languages/supports/indentationLineProcessor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 9edcf078cb2f2..6acbcd606088f 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -143,7 +143,7 @@ export class IndentationContextProcessor { return processedLine; } - private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens) { + private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): string { let processedPreviousLine = ''; if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { // This is not the first line and the entire line belongs to this mode @@ -194,7 +194,7 @@ class IndentationLineProcessor { const currentIndentation = strings.getLeadingWhitespace(currentLine); const currentTokens = this.model.tokenization.getLineTokens(lineNumber); const indentationDifference = newIndentation.length - currentIndentation.length; - const newLine = newIndentation + currentLine.substring(currentIndentation.length);; + const newLine = newIndentation + currentLine.substring(currentIndentation.length); const newTokens = currentTokens.shiftTokenOffsetsBy(indentationDifference, newLine); return this.getProcessedLineForLineAndTokens(newLine, newTokens); } From 45b5be80d536042b1bd76857e5170c95a667d065 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 23 Apr 2024 16:14:39 +0200 Subject: [PATCH 24/47] placing the indentation addition into the indentation line processor --- .../supports/indentationLineProcessor.ts | 42 +++++++------------ 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 6acbcd606088f..960c269b85e03 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -36,7 +36,7 @@ export class ProcessedIndentRulesSupport { * Apply the new indentation and return whether the indentation level should be increased after the given line number */ public shouldIncrease(lineNumber: number, newIndentation?: string): boolean { - const processedLine = this._processLine(lineNumber, newIndentation); + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIncrease(processedLine); } @@ -44,7 +44,7 @@ export class ProcessedIndentRulesSupport { * Apply the new indentation and return whether the indentation level should be decreased after the given line number */ public shouldDecrease(lineNumber: number, newIndentation?: string): boolean { - const processedLine = this._processLine(lineNumber, newIndentation); + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldDecrease(processedLine); } @@ -52,7 +52,7 @@ export class ProcessedIndentRulesSupport { * Apply the new indentation and return whether the indentation level should remain unchanged at the given line number */ public shouldIgnore(lineNumber: number, newIndentation?: string): boolean { - const processedLine = this._processLine(lineNumber, newIndentation); + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIgnore(processedLine); } @@ -60,19 +60,10 @@ export class ProcessedIndentRulesSupport { * Apply the new indentation and return whether the indentation level should increase on the line after the given line number */ public shouldIndentNextLine(lineNumber: number, newIndentation?: string): boolean { - const processedLine = this._processLine(lineNumber, newIndentation); + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); return this._indentRulesSupport.shouldIndentNextLine(processedLine); } - private _processLine(lineNumber: number, newIndentation?: string) { - let processedLine: string; - if (newIndentation === undefined) { - processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber); - } else { - processedLine = this._indentationLineProcessor.getProcessedLineWithIndentation(lineNumber, newIndentation); - } - return processedLine; - } } /** @@ -177,26 +168,23 @@ class IndentationLineProcessor { ) { } /** - * Get the processed line for the given line number. Remove the language configuration brackets from the regex, string and comment tokens. + * Get the processed line for the given line number and potentially adjust the indentation level. Remove the language configuration brackets from the regex, string and comment tokens. */ - getProcessedLine(lineNumber: number): string { + getProcessedLine(lineNumber: number, newIndentation?: string): string { const lineContent = this.model.getLineContent(lineNumber); const tokens = this.model.tokenization.getLineTokens(lineNumber); const processedLine = this.getProcessedLineForLineAndTokens(lineContent, tokens); - return processedLine; + const adjustedProcessedLine = this._adjustIndentation(processedLine, newIndentation); + return adjustedProcessedLine; } - /** - * Replace the indentation of the line at the given line number with the new indentation and process the line - remove the language configuration brackets from the regex, string and comment tokens. - */ - getProcessedLineWithIndentation(lineNumber: number, newIndentation: string): string { - const currentLine = this.model.getLineContent(lineNumber); - const currentIndentation = strings.getLeadingWhitespace(currentLine); - const currentTokens = this.model.tokenization.getLineTokens(lineNumber); - const indentationDifference = newIndentation.length - currentIndentation.length; - const newLine = newIndentation + currentLine.substring(currentIndentation.length); - const newTokens = currentTokens.shiftTokenOffsetsBy(indentationDifference, newLine); - return this.getProcessedLineForLineAndTokens(newLine, newTokens); + private _adjustIndentation(line: string, newIndentation?: string): string { + if (newIndentation === undefined) { + return line; + } + const currentIndentation = strings.getLeadingWhitespace(line); + const adjustedLine = newIndentation + line.substring(currentIndentation.length); + return adjustedLine; } /** From 7d37ff6b726ecbb4424b4e2316db59f7cae5614e Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 23 Apr 2024 17:01:58 +0200 Subject: [PATCH 25/47] defining a for each method on the line tokens --- .../supports/indentationLineProcessor.ts | 51 +++++++------------ src/vs/editor/common/tokens/lineTokens.ts | 43 +++++++++------- .../editor/test/common/core/testLineToken.ts | 17 ++----- 3 files changed, 45 insertions(+), 66 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 960c269b85e03..eb2b176784b82 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -11,7 +11,7 @@ import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/langu import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; -import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { StandardTokenType, TokenMetadata } from 'vs/editor/common/encodedTokenAttributes'; /** * This class is a wrapper class around {@link IndentRulesSupport}. @@ -109,9 +109,8 @@ export class IndentationContextProcessor { const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const scopedLineContent = scopedLineTokens.getLineContent(); - const slicedLine = scopedLineContent.substring(0, columnIndexWithinScope); const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedTokens) ?? scopedLineContent.substring(0, columnIndexWithinScope); return processedLine; } @@ -128,9 +127,8 @@ export class IndentationContextProcessor { const scopedLineContent = scopedLineTokens.getLineContent(); const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length + 1; - const slicedLine = scopedLineContent.substring(columnIndexWithinScope); const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedLine, slicedTokens); + const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedTokens) ?? scopedLineContent.substring(columnIndexWithinScope); return processedLine; } @@ -149,7 +147,7 @@ export class IndentationContextProcessor { const firstCharacterOffset = previousLineScopedLineTokens.firstCharOffset; const lastCharacterOffset = firstCharacterOffset + previousLine.length; const previousLineTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLine, previousLineTokens); + processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLineTokens) ?? previousLine; } } return processedPreviousLine; @@ -171,13 +169,15 @@ class IndentationLineProcessor { * Get the processed line for the given line number and potentially adjust the indentation level. Remove the language configuration brackets from the regex, string and comment tokens. */ getProcessedLine(lineNumber: number, newIndentation?: string): string { - const lineContent = this.model.getLineContent(lineNumber); const tokens = this.model.tokenization.getLineTokens(lineNumber); - const processedLine = this.getProcessedLineForLineAndTokens(lineContent, tokens); + const processedLine = this.getProcessedLineForLineAndTokens(tokens) ?? this.model.getLineContent(lineNumber); const adjustedProcessedLine = this._adjustIndentation(processedLine, newIndentation); return adjustedProcessedLine; } + /** + * Adjust the indentation of the given line to the given new indentation + */ private _adjustIndentation(line: string, newIndentation?: string): string { if (newIndentation === undefined) { return line; @@ -190,24 +190,9 @@ class IndentationLineProcessor { /** * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. */ - getProcessedLineForLineAndTokens(line: string, tokens: IViewLineTokens): string { + getProcessedLineForLineAndTokens(tokens: IViewLineTokens): string | undefined { // Utility functions - const removeBracketsFromTokenWithIndexWithinLine = (tokenIndex: number, offset: number, line: string): { processedOffset: number, processedLine: string } => { - const result = removeBracketsFromTokenWithIndex(tokenIndex); - const processedOffset = offset - (result.tokenText.length - result.processedTokenText.length); - const lineBeforeToken = line.substring(0, offset + result.tokenInitialStartOffset); - const lineAfterToken = line.substring(offset + result.tokenInitialEndOffset); - const processedLine = lineBeforeToken + result.processedTokenText + lineAfterToken; - return { processedOffset, processedLine }; - }; - const removeBracketsFromTokenWithIndex = (tokenIndex: number): { tokenText: string; processedTokenText: string; tokenInitialStartOffset: number; tokenInitialEndOffset: number } => { - const tokenInitialStartOffset = tokens.getStartOffset(tokenIndex); - const tokenInitialEndOffset = tokens.getEndOffset(tokenIndex); - const tokenText = line.substring(tokenInitialStartOffset, tokenInitialEndOffset); - const processedTokenText = removeBracketsFromText(tokenText); - return { tokenText, processedTokenText, tokenInitialStartOffset, tokenInitialEndOffset }; - } const removeBracketsFromText = (line: string): string => { let processedLine = line; openBrackets.forEach((bracket) => { @@ -235,25 +220,23 @@ class IndentationLineProcessor { const languageId = tokens.getLanguageId(0); const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; if (!brackets) { - return line; + return undefined; } const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - let processedOffset = 0; - let processedLine = line; - - for (let i = 0; i < tokens.getCount(); i++) { - const standardTokenType = tokens.getStandardTokenType(i); + let processedLine = ''; + tokens.forEach((text: string, metadata: number) => { + const standardTokenType = TokenMetadata.getTokenType(metadata); if (standardTokenType === StandardTokenType.String || standardTokenType === StandardTokenType.RegEx || standardTokenType === StandardTokenType.Comment ) { - const result = removeBracketsFromTokenWithIndexWithinLine(i, processedOffset, processedLine); - processedOffset = result.processedOffset; - processedLine = result.processedLine; + processedLine += removeBracketsFromText(text); + } else { + processedLine += text; } - } + }); return processedLine; } } diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index cf74a35bccab8..033500175c90b 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -10,7 +10,6 @@ export interface IViewLineTokens { equals(other: IViewLineTokens): boolean; getCount(): number; getForeground(tokenIndex: number): ColorId; - getStartOffset(tokenIndex: number): number; getEndOffset(tokenIndex: number): number; getClassName(tokenIndex: number): string; getInlineStyle(tokenIndex: number, colorMap: string[]): string; @@ -19,7 +18,7 @@ export interface IViewLineTokens { getLineContent(): string; getMetadata(tokenIndex: number): number; getLanguageId(tokenIndex: number): string; - getStandardTokenType(tokenIndex: number): StandardTokenType; + forEach(f: (text: string, metadata: number) => void): void; } export class LineTokens implements IViewLineTokens { @@ -183,15 +182,6 @@ export class LineTokens implements IViewLineTokens { return low; } - public shiftTokenOffsetsBy(delta: number, newTokenText: string): LineTokens { - const newTokens = new Array(); - for (let i = 0; i < this.getCount(); i++) { - newTokens.push(this.getEndOffset(i) + delta); - newTokens.push(this.getMetadata(i)); - } - return new LineTokens(new Uint32Array(newTokens), newTokenText, this._languageIdCodec); - } - /** * @pure * @param insertTokens Must be sorted by offset. @@ -238,6 +228,22 @@ export class LineTokens implements IViewLineTokens { return new LineTokens(new Uint32Array(newTokens), text, this._languageIdCodec); } + + public getTextForTokenIndex(tokenIndex: number): string { + const startOffset = this.getStartOffset(tokenIndex); + const endOffset = this.getEndOffset(tokenIndex); + const text = this._text.substring(startOffset, endOffset); + return text; + } + + public forEach(f: (text: string, metadata: number) => void): void { + const tokenCount = this.getCount(); + for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) { + const metadata = this.getMetadata(tokenIndex); + const text = this.getTextForTokenIndex(tokenIndex); + f(text, metadata); + } + } } class SliceLineTokens implements IViewLineTokens { @@ -299,11 +305,6 @@ class SliceLineTokens implements IViewLineTokens { return this._source.getForeground(this._firstTokenIndex + tokenIndex); } - public getStartOffset(tokenIndex: number): number { - const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); - return Math.max(this._startOffset, tokenStartOffset) - this._startOffset + this._deltaOffset; - } - public getEndOffset(tokenIndex: number): number { const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); return Math.min(this._endOffset, tokenEndOffset) - this._startOffset + this._deltaOffset; @@ -325,7 +326,13 @@ class SliceLineTokens implements IViewLineTokens { return this._source.findTokenIndexAtOffset(offset + this._startOffset - this._deltaOffset) - this._firstTokenIndex; } - public getStandardTokenType(tokenIndex: number): StandardTokenType { - return this._source.getStandardTokenType(this._firstTokenIndex + tokenIndex); + public forEach(f: (text: string, metadata: number) => void): void { + const tokenCount = this.getCount(); + const lastTokenIndex = this._firstTokenIndex + tokenCount; + for (let tokenIndex = this._startOffset; tokenIndex < lastTokenIndex; tokenIndex++) { + const metadata = this._source.getMetadata(tokenIndex); + const text = this._source.getTextForTokenIndex(tokenIndex) + f(text, metadata); + } } } diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index cf7e781531df4..b526cd3c14bd5 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { ColorId, TokenMetadata, ITokenPresentation, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ColorId, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; /** * A token on a line. @@ -38,10 +38,6 @@ export class TestLineToken { return TokenMetadata.getPresentationFromMetadata(this._metadata); } - public getStandardTokenType(): StandardTokenType { - return TokenMetadata.getTokenType(this._metadata) - } - private static _equals(a: TestLineToken, b: TestLineToken): boolean { return ( a.endIndex === b.endIndex @@ -87,13 +83,6 @@ export class TestLineTokens implements IViewLineTokens { return this._actual[tokenIndex].getForeground(); } - public getStartOffset(tokenIndex: number): number { - if (tokenIndex > 0) { - return this._actual[tokenIndex - 1].endIndex + 1; - } - return 0; - } - public getEndOffset(tokenIndex: number): number { return this._actual[tokenIndex].endIndex; } @@ -126,8 +115,8 @@ export class TestLineTokens implements IViewLineTokens { throw new Error('Method not implemented.'); } - public getStandardTokenType(tokenIndex: number): StandardTokenType { - return this._actual[tokenIndex].getStandardTokenType(); + public forEach(f: (text: string, metadata: number) => void): void { + throw new Error('Not implemented'); } } From 51c97b505d924c5476d7736dd99148d839fd9c64 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 23 Apr 2024 17:32:05 +0200 Subject: [PATCH 26/47] polishing the code --- .../supports/indentationLineProcessor.ts | 42 +++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index eb2b176784b82..3ecc8d4a8bbdd 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -108,9 +108,8 @@ export class IndentationContextProcessor { const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const scopedLineContent = scopedLineTokens.getLineContent(); const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedTokens) ?? scopedLineContent.substring(0, columnIndexWithinScope); + const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokens); return processedLine; } @@ -128,7 +127,7 @@ export class IndentationContextProcessor { const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length + 1; const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(slicedTokens) ?? scopedLineContent.substring(columnIndexWithinScope); + const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokens); return processedLine; } @@ -143,11 +142,11 @@ export class IndentationContextProcessor { const previousLineScopedLineTokens = createScopedLineTokens(lineTokens, column); if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { // The line above ends with text belonging to the same mode - const previousLine = previousLineScopedLineTokens.getLineContent(); + const previousScopedLine = previousLineScopedLineTokens.getLineContent(); const firstCharacterOffset = previousLineScopedLineTokens.firstCharOffset; - const lastCharacterOffset = firstCharacterOffset + previousLine.length; + const lastCharacterOffset = firstCharacterOffset + previousScopedLine.length; const previousLineTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - processedPreviousLine = this.indentationLineProcessor.getProcessedLineForLineAndTokens(previousLineTokens) ?? previousLine; + processedPreviousLine = this.indentationLineProcessor.getProcessedLineForTokens(previousLineTokens); } } return processedPreviousLine; @@ -169,28 +168,27 @@ class IndentationLineProcessor { * Get the processed line for the given line number and potentially adjust the indentation level. Remove the language configuration brackets from the regex, string and comment tokens. */ getProcessedLine(lineNumber: number, newIndentation?: string): string { - const tokens = this.model.tokenization.getLineTokens(lineNumber); - const processedLine = this.getProcessedLineForLineAndTokens(tokens) ?? this.model.getLineContent(lineNumber); - const adjustedProcessedLine = this._adjustIndentation(processedLine, newIndentation); - return adjustedProcessedLine; - } - /** - * Adjust the indentation of the given line to the given new indentation - */ - private _adjustIndentation(line: string, newIndentation?: string): string { - if (newIndentation === undefined) { - return line; + // Utility function + const adjustIndentation = (line: string, newIndentation: string): string => { + const currentIndentation = strings.getLeadingWhitespace(line); + const adjustedLine = newIndentation + line.substring(currentIndentation.length); + return adjustedLine; } - const currentIndentation = strings.getLeadingWhitespace(line); - const adjustedLine = newIndentation + line.substring(currentIndentation.length); - return adjustedLine; + + // Main code + const tokens = this.model.tokenization.getLineTokens(lineNumber); + let processedLine = this.getProcessedLineForTokens(tokens); + if (newIndentation !== undefined) { + processedLine = adjustIndentation(processedLine, newIndentation); + } + return processedLine; } /** * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. */ - getProcessedLineForLineAndTokens(tokens: IViewLineTokens): string | undefined { + getProcessedLineForTokens(tokens: IViewLineTokens): string { // Utility functions const removeBracketsFromText = (line: string): string => { @@ -220,7 +218,7 @@ class IndentationLineProcessor { const languageId = tokens.getLanguageId(0); const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; if (!brackets) { - return undefined; + return tokens.getLineContent(); } const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); From 2d2064560a0db845b3412d4e7c4161535ce22572 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 23 Apr 2024 17:51:48 +0200 Subject: [PATCH 27/47] trim the text if token offsets overflow --- src/vs/editor/common/tokens/lineTokens.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 033500175c90b..94d23affeee75 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -328,10 +328,22 @@ class SliceLineTokens implements IViewLineTokens { public forEach(f: (text: string, metadata: number) => void): void { const tokenCount = this.getCount(); + const firstTokenIndex = this._firstTokenIndex; const lastTokenIndex = this._firstTokenIndex + tokenCount; - for (let tokenIndex = this._startOffset; tokenIndex < lastTokenIndex; tokenIndex++) { + for (let tokenIndex = firstTokenIndex; tokenIndex < lastTokenIndex; tokenIndex++) { const metadata = this._source.getMetadata(tokenIndex); - const text = this._source.getTextForTokenIndex(tokenIndex) + let text = this._source.getTextForTokenIndex(tokenIndex) + // May have to trim the text if overflows + const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); + const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); + if (tokenStartOffset < this._startOffset) { + const offsetDifference = this._startOffset - tokenStartOffset; + text = text.substring(offsetDifference); + } + if (tokenEndOffset > this._endOffset) { + const offsetDifference = tokenEndOffset - this._endOffset; + text = text.substring(0, text.length - offsetDifference); + } f(text, metadata); } } From c4cf3bdc5bb9b47d9675a08a27df3da3fab9cb19 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Wed, 24 Apr 2024 09:49:13 +0200 Subject: [PATCH 28/47] changing the line position --- src/vs/editor/common/tokens/lineTokens.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 94d23affeee75..961018f960634 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -332,10 +332,9 @@ class SliceLineTokens implements IViewLineTokens { const lastTokenIndex = this._firstTokenIndex + tokenCount; for (let tokenIndex = firstTokenIndex; tokenIndex < lastTokenIndex; tokenIndex++) { const metadata = this._source.getMetadata(tokenIndex); - let text = this._source.getTextForTokenIndex(tokenIndex) - // May have to trim the text if overflows const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); + let text = this._source.getTextForTokenIndex(tokenIndex) if (tokenStartOffset < this._startOffset) { const offsetDifference = this._startOffset - tokenStartOffset; text = text.substring(offsetDifference); From 55930395c869d186a3bd5c3ddc117281006bbb7f Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 25 Apr 2024 15:33:44 +0200 Subject: [PATCH 29/47] polishing the code --- src/vs/editor/common/languages/supports.ts | 4 + .../supports/indentationLineProcessor.ts | 76 ++++++++++++------- src/vs/editor/common/tokens/lineTokens.ts | 47 +++++++----- .../editor/test/common/core/testLineToken.ts | 16 +++- 4 files changed, 92 insertions(+), 51 deletions(-) diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 8fdfa17bb5132..bf434a95a241b 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -78,6 +78,10 @@ export class ScopedLineTokens { public getStandardTokenType(tokenIndex: number): StandardTokenType { return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } + + public doesScopeStartAtOffsetZero(): boolean { + return this.firstCharOffset === 0; + } } const enum IgnoreBracketsInTokens { diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 3ecc8d4a8bbdd..3601e959e6761 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -11,7 +11,7 @@ import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/langu import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; -import { StandardTokenType, TokenMetadata } from 'vs/editor/common/encodedTokenAttributes'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; /** * This class is a wrapper class around {@link IndentRulesSupport}. @@ -108,8 +108,8 @@ export class IndentationContextProcessor { const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokens); + const slicedTokensBefore = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokensBefore); return processedLine; } @@ -126,30 +126,44 @@ export class IndentationContextProcessor { const scopedLineContent = scopedLineTokens.getLineContent(); const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length + 1; - const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokens); + const slicedTokensAfter = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokensAfter); return processedLine; } private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): string { - let processedPreviousLine = ''; - if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { - // This is not the first line and the entire line belongs to this mode - const previousLineNumber = range.startLineNumber - 1; - this.model.tokenization.forceTokenization(previousLineNumber); - const lineTokens = this.model.tokenization.getLineTokens(previousLineNumber); - const column = this.model.getLineMaxColumn(previousLineNumber) - 1; - const previousLineScopedLineTokens = createScopedLineTokens(lineTokens, column); - if (previousLineScopedLineTokens.languageId === scopedLineTokens.languageId) { - // The line above ends with text belonging to the same mode - const previousScopedLine = previousLineScopedLineTokens.getLineContent(); - const firstCharacterOffset = previousLineScopedLineTokens.firstCharOffset; - const lastCharacterOffset = firstCharacterOffset + previousScopedLine.length; - const previousLineTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - processedPreviousLine = this.indentationLineProcessor.getProcessedLineForTokens(previousLineTokens); - } + + // Utility function + const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { + this.model.tokenization.forceTokenization(lineNumber); + const lineTokens = this.model.tokenization.getLineTokens(lineNumber); + const endColumnOfLine = this.model.getLineMaxColumn(lineNumber) - 1; + const scopedLineTokensAtEndColumn = createScopedLineTokens(lineTokens, endColumnOfLine); + return scopedLineTokensAtEndColumn; } - return processedPreviousLine; + + // Main code + const isFirstLine = range.startLineNumber === 1; + if (isFirstLine) { + return ''; + } + const canScopeExtendOnPreviousLine = scopedLineTokens.doesScopeStartAtOffsetZero(); + if (!canScopeExtendOnPreviousLine) { + return ''; + } + const previousLineNumber = range.startLineNumber - 1; + const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); + const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; + if (!doesLanguageContinueOnPreviousLine) { + return ''; + } + const previousScopedLine = scopedLineTokensAtEndColumnOfPreviousLine.getLineContent(); + const firstCharacterOffset = scopedLineTokensAtEndColumnOfPreviousLine.firstCharOffset; + const lastCharacterOffset = firstCharacterOffset + previousScopedLine.length; + const previousLineTokens = this.model.tokenization.getLineTokens(previousLineNumber); + const previousSlicedLineTokens = previousLineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedPreviousScopedLine = this.indentationLineProcessor.getProcessedLineForTokens(previousSlicedLineTokens); + return processedPreviousScopedLine; } } @@ -165,7 +179,8 @@ class IndentationLineProcessor { ) { } /** - * Get the processed line for the given line number and potentially adjust the indentation level. Remove the language configuration brackets from the regex, string and comment tokens. + * Get the processed line for the given line number and potentially adjust the indentation level. + * Remove the language configuration brackets from the regex, string and comment tokens. */ getProcessedLine(lineNumber: number, newIndentation?: string): string { @@ -191,6 +206,11 @@ class IndentationLineProcessor { getProcessedLineForTokens(tokens: IViewLineTokens): string { // Utility functions + const isTokenTypeToProcess = (tokenType: StandardTokenType): boolean => { + return tokenType === StandardTokenType.String + || tokenType === StandardTokenType.RegEx + || tokenType === StandardTokenType.Comment; + } const removeBracketsFromText = (line: string): string => { let processedLine = line; openBrackets.forEach((bracket) => { @@ -224,12 +244,10 @@ class IndentationLineProcessor { const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); let processedLine = ''; - tokens.forEach((text: string, metadata: number) => { - const standardTokenType = TokenMetadata.getTokenType(metadata); - if (standardTokenType === StandardTokenType.String - || standardTokenType === StandardTokenType.RegEx - || standardTokenType === StandardTokenType.Comment - ) { + tokens.forEach((tokenIndex: number) => { + const tokenType = tokens.getStandardTokenType(tokenIndex); + const text = tokens.getTokenText(tokenIndex); + if (isTokenTypeToProcess(tokenType)) { processedLine += removeBracketsFromText(text); } else { processedLine += text; diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 961018f960634..b17be218ca2c8 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -9,6 +9,7 @@ import { FontStyle, ColorId, StandardTokenType, MetadataConsts, TokenMetadata, I export interface IViewLineTokens { equals(other: IViewLineTokens): boolean; getCount(): number; + getStandardTokenType(tokenIndex: number): StandardTokenType; getForeground(tokenIndex: number): ColorId; getEndOffset(tokenIndex: number): number; getClassName(tokenIndex: number): string; @@ -18,7 +19,8 @@ export interface IViewLineTokens { getLineContent(): string; getMetadata(tokenIndex: number): number; getLanguageId(tokenIndex: number): string; - forEach(f: (text: string, metadata: number) => void): void; + getTokenText(tokenIndex: number): string; + forEach(callback: (tokenIndex: number) => void): void; } export class LineTokens implements IViewLineTokens { @@ -229,19 +231,17 @@ export class LineTokens implements IViewLineTokens { return new LineTokens(new Uint32Array(newTokens), text, this._languageIdCodec); } - public getTextForTokenIndex(tokenIndex: number): string { + public getTokenText(tokenIndex: number): string { const startOffset = this.getStartOffset(tokenIndex); const endOffset = this.getEndOffset(tokenIndex); const text = this._text.substring(startOffset, endOffset); return text; } - public forEach(f: (text: string, metadata: number) => void): void { + public forEach(callback: (tokenIndex: number) => void): void { const tokenCount = this.getCount(); for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) { - const metadata = this.getMetadata(tokenIndex); - const text = this.getTextForTokenIndex(tokenIndex); - f(text, metadata); + callback(tokenIndex); } } } @@ -301,6 +301,10 @@ class SliceLineTokens implements IViewLineTokens { return this._tokensCount; } + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._source.getStandardTokenType(this._firstTokenIndex + tokenIndex); + } + public getForeground(tokenIndex: number): ColorId { return this._source.getForeground(this._firstTokenIndex + tokenIndex); } @@ -326,24 +330,27 @@ class SliceLineTokens implements IViewLineTokens { return this._source.findTokenIndexAtOffset(offset + this._startOffset - this._deltaOffset) - this._firstTokenIndex; } - public forEach(f: (text: string, metadata: number) => void): void { + public getTokenText(tokenIndex: number): string { + const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); + const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); + let text = this._source.getTokenText(tokenIndex) + if (tokenStartOffset < this._startOffset) { + const offsetDifference = this._startOffset - tokenStartOffset; + text = text.substring(offsetDifference); + } + if (tokenEndOffset > this._endOffset) { + const offsetDifference = tokenEndOffset - this._endOffset; + text = text.substring(0, text.length - offsetDifference); + } + return text; + } + + public forEach(callback: (tokenIndex: number) => void): void { const tokenCount = this.getCount(); const firstTokenIndex = this._firstTokenIndex; const lastTokenIndex = this._firstTokenIndex + tokenCount; for (let tokenIndex = firstTokenIndex; tokenIndex < lastTokenIndex; tokenIndex++) { - const metadata = this._source.getMetadata(tokenIndex); - const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); - const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); - let text = this._source.getTextForTokenIndex(tokenIndex) - if (tokenStartOffset < this._startOffset) { - const offsetDifference = this._startOffset - tokenStartOffset; - text = text.substring(offsetDifference); - } - if (tokenEndOffset > this._endOffset) { - const offsetDifference = tokenEndOffset - this._endOffset; - text = text.substring(0, text.length - offsetDifference); - } - f(text, metadata); + callback(tokenIndex); } } } diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index b526cd3c14bd5..c10558800de5a 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { ColorId, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; +import { ColorId, TokenMetadata, ITokenPresentation, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; /** * A token on a line. @@ -22,6 +22,10 @@ export class TestLineToken { this._metadata = metadata; } + public getStandardTokenType(): StandardTokenType { + return TokenMetadata.getTokenType(this._metadata); + } + public getForeground(): ColorId { return TokenMetadata.getForeground(this._metadata); } @@ -79,6 +83,10 @@ export class TestLineTokens implements IViewLineTokens { return this._actual.length; } + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._actual[tokenIndex].getStandardTokenType(); + } + public getForeground(tokenIndex: number): ColorId { return this._actual[tokenIndex].getForeground(); } @@ -115,7 +123,11 @@ export class TestLineTokens implements IViewLineTokens { throw new Error('Method not implemented.'); } - public forEach(f: (text: string, metadata: number) => void): void { + public getTokenText(tokenIndex: number): string { + throw new Error('Method not implemented.'); + } + + public forEach(callback: (tokenIndex: number) => void): void { throw new Error('Not implemented'); } } From 48122bad1e498990b4827a307353c6d78fcf05b6 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 25 Apr 2024 15:55:43 +0200 Subject: [PATCH 30/47] polishing the code --- src/vs/editor/common/languages/autoIndent.ts | 5 ++++- .../supports/indentationLineProcessor.ts | 18 +++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index d76bb486c52d9..90d5e05c5da5f 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -446,5 +446,8 @@ export function getIndentMetadata( export function isWithinEmbeddedLanguage(model: ITextModel, position: Position): boolean { const lineTokens = model.tokenization.getLineTokens(position.lineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); - return scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId; + const doesScopeStartAtOffsetZero = scopedLineTokens.doesScopeStartAtOffsetZero(); + const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; + const isWithinEmbeddedLanguage = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; + return isWithinEmbeddedLanguage; }; diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 3601e959e6761..cf5e569f631bd 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -141,9 +141,18 @@ export class IndentationContextProcessor { const scopedLineTokensAtEndColumn = createScopedLineTokens(lineTokens, endColumnOfLine); return scopedLineTokensAtEndColumn; } + const getSlicedLineTokensForScopeAtLine = (scopedLineTokens: ScopedLineTokens, lineNumber: number): IViewLineTokens => { + const initialLine = this.model.tokenization.getLineTokens(lineNumber); + const scopedLine = scopedLineTokens.getLineContent(); + const firstCharacterOffset = scopedLineTokens.firstCharOffset; + const lastCharacterOffset = firstCharacterOffset + scopedLine.length; + const slicedLineTokens = initialLine.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + return slicedLineTokens; + } // Main code - const isFirstLine = range.startLineNumber === 1; + const previousLineNumber = range.startLineNumber - 1; + const isFirstLine = previousLineNumber === 0; if (isFirstLine) { return ''; } @@ -151,17 +160,12 @@ export class IndentationContextProcessor { if (!canScopeExtendOnPreviousLine) { return ''; } - const previousLineNumber = range.startLineNumber - 1; const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; if (!doesLanguageContinueOnPreviousLine) { return ''; } - const previousScopedLine = scopedLineTokensAtEndColumnOfPreviousLine.getLineContent(); - const firstCharacterOffset = scopedLineTokensAtEndColumnOfPreviousLine.firstCharOffset; - const lastCharacterOffset = firstCharacterOffset + previousScopedLine.length; - const previousLineTokens = this.model.tokenization.getLineTokens(previousLineNumber); - const previousSlicedLineTokens = previousLineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); const processedPreviousScopedLine = this.indentationLineProcessor.getProcessedLineForTokens(previousSlicedLineTokens); return processedPreviousScopedLine; } From da798e6ea3c17a8b58e509bd796d6f3c5bf2bcf9 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 25 Apr 2024 16:13:59 +0200 Subject: [PATCH 31/47] removing the 1 from the end character offset calculations --- .../common/languages/supports/indentationLineProcessor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index cf5e569f631bd..dec0e4daf7d38 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -124,8 +124,8 @@ export class IndentationContextProcessor { lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); } const scopedLineContent = scopedLineTokens.getLineContent(); - const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope + 1; - const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length + 1; + const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; + const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length; const slicedTokensAfter = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokensAfter); return processedLine; From 86f1fae870e3babd1456fe7bf3abc8c8c46ab842 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 13 May 2024 12:23:33 +0200 Subject: [PATCH 32/47] fixing compile errors --- .../common/languages/supports/indentationLineProcessor.ts | 2 +- .../editor/contrib/indentation/test/browser/indentation.test.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index dec0e4daf7d38..0a3f74351d839 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -133,7 +133,7 @@ export class IndentationContextProcessor { private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): string { - // Utility function + // Utility functions const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { this.model.tokenization.forceTokenization(lineNumber); const lineTokens = this.model.tokenization.getLineTokens(lineNumber); diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index bfe9952b233fd..a0d6dd2b2f88f 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -5,7 +5,6 @@ import * as assert from 'assert'; import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; -import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createTextModel } from 'vs/editor/test/common/testTextModel'; From feb1a316dcba8af22522128df3a74b0c6227b9e2 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 13 May 2024 12:35:16 +0200 Subject: [PATCH 33/47] removing the changes to the autoindent.test.ts file, this will be done in a separate PR --- .../codeEditor/test/node/autoindent.test.ts | 66 +------------------ 1 file changed, 2 insertions(+), 64 deletions(-) diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index ff66287513dd8..cc9316468bc41 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -18,10 +18,6 @@ import { IRange } from 'vs/editor/common/core/range'; import { ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { trimTrailingWhitespace } from 'vs/editor/common/commands/trimTrailingWhitespaceCommand'; import { execSync } from 'child_process'; -import { ILanguageService } from 'vs/editor/common/languages/language'; -import { EncodedTokenizationResult, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; -import { NullState } from 'vs/editor/common/languages/nullTokenize'; -import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; function getIRange(range: IRange): IRange { return { @@ -36,12 +32,7 @@ const enum LanguageId { TypeScript = 'ts-test' } -function registerLanguage(instantiationService: TestInstantiationService, languageId: LanguageId): IDisposable { - const languageService = instantiationService.get(ILanguageService) - return languageService.registerLanguage({ id: languageId }); -} - -function registerLanguageConfiguration(languageConfigurationService: ILanguageConfigurationService, languageId: LanguageId): IDisposable { +function registerLanguage(languageConfigurationService: ILanguageConfigurationService, languageId: LanguageId): IDisposable { let configPath: string; switch (languageId) { case LanguageId.TypeScript: @@ -56,33 +47,6 @@ function registerLanguageConfiguration(languageConfigurationService: ILanguageCo return languageConfigurationService.register(languageId, languageConfig); } -interface TokenData { - startIndex: number; - value: number -} - -function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: TokenData[][], languageId: string): IDisposable { - let lineIndex = 0; - const languageService = instantiationService.get(ILanguageService); - const tokenizationSupport: ITokenizationSupport = { - getInitialState: () => NullState, - tokenize: undefined!, - tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { - const tokensOnLine = tokens[lineIndex++]; - const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId); - const result = new Uint32Array(2 * tokensOnLine.length); - for (let i = 0; i < tokensOnLine.length; i++) { - result[2 * i] = tokensOnLine[i].startIndex; - result[2 * i + 1] = - ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (tokensOnLine[i].value << MetadataConsts.TOKEN_TYPE_OFFSET)); - } - return new EncodedTokenizationResult(result, state); - } - }; - return TokenizationRegistry.register(languageId, tokenizationSupport); -} - suite('Auto-Reindentation - TypeScript/JavaScript', () => { const languageId = LanguageId.TypeScript; @@ -94,10 +58,8 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { setup(() => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); - disposables.add(instantiationService); - disposables.add(registerLanguage(instantiationService, languageId)); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); - disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); + disposables.add(registerLanguage(languageConfigurationService, languageId)); }); teardown(() => { @@ -198,19 +160,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const foo = `{`;', ' ', ].join('\n'); - const tokens: TokenData[][] = [ - [ - { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, - { startIndex: 9, value: 0 }, { startIndex: 10, value: 0 }, { startIndex: 11, value: 0 }, - { startIndex: 12, value: 2 }, { startIndex: 13, value: 2 }, { startIndex: 14, value: 2 }, - { startIndex: 15, value: 0 }, { startIndex: 16, value: 0 } - ], - [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] - ]; - disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - model.tokenization.forceTokenization(1); - model.tokenization.forceTokenization(2); const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; @@ -308,19 +258,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { 'const r = /{/;', ' ', ].join('\n'); - const tokens: TokenData[][] = [ - [ - { startIndex: 0, value: 0 }, { startIndex: 5, value: 0 }, { startIndex: 6, value: 0 }, - { startIndex: 7, value: 0 }, { startIndex: 8, value: 0 }, { startIndex: 9, value: 3 }, - { startIndex: 10, value: 3 }, { startIndex: 11, value: 3 }, { startIndex: 12, value: 3 }, - { startIndex: 13, value: 0 }, { startIndex: 14, value: 0 } - ], - [{ startIndex: 0, value: 0 }, { startIndex: 4, value: 0 }] - ]; - disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - model.tokenization.forceTokenization(1); - model.tokenization.forceTokenization(2); const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; From 8770945169cf5bb3f81dd0eb9d31087da8b3c023 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 13 May 2024 16:33:25 +0200 Subject: [PATCH 34/47] adding tokenization support into the test --- .../test/browser/indentation.test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index a0d6dd2b2f88f..1097b1bb316f4 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -115,11 +115,6 @@ interface StandardTokenTypeData { standardTokenType: StandardTokenType; } -interface StandardTokenTypeData { - startIndex: number; - standardTokenType: StandardTokenType; -} - function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: StandardTokenTypeData[][], languageId: string): IDisposable { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); @@ -1026,6 +1021,12 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { disposables.add(registerLanguage(instantiationService, languageId)); + const tokens: StandardTokenTypeData[][] = [ + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }] + ]; + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(2, 23, 2, 23)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ @@ -1417,6 +1418,14 @@ suite('Auto Indent On Type - PHP', () => { withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { disposables.add(registerLanguage(instantiationService, languageId)); + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 13, standardTokenType: StandardTokenType.String }, + { startIndex: 16, standardTokenType: StandardTokenType.Other }, + ] + ]; + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(1, 54, 1, 54)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ From 0ddda76419d5354ea255f6a59292ea98b49caa73 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 11:28:27 +0200 Subject: [PATCH 35/47] adding code in order to include the modified line tokens --- .../widget/codeEditor/codeEditorWidget.ts | 3 + .../codeEditor/embeddedCodeEditorWidget.ts | 4 +- src/vs/editor/common/commands/shiftCommand.ts | 4 +- .../common/cursor/cursorTypeOperations.ts | 16 ++-- src/vs/editor/common/cursorCommon.ts | 2 + src/vs/editor/common/languages/autoIndent.ts | 51 +++++++------ src/vs/editor/common/languages/enterAction.ts | 9 ++- .../supports/indentationLineProcessor.ts | 75 ++++++++++++------- .../editor/common/viewModel/viewModelImpl.ts | 12 +-- .../indentation/browser/indentation.ts | 12 ++- .../contrib/indentation/common/indentation.ts | 5 +- .../browser/linesOperations.ts | 4 +- .../browser/moveLinesCommand.ts | 11 ++- .../test/browser/moveLinesCommand.test.ts | 33 +++++--- .../browser/standaloneCodeEditor.ts | 5 +- .../browser/commands/shiftCommand.test.ts | 21 +++--- .../test/browser/viewModel/testViewModel.ts | 4 +- .../codeEditor/test/node/autoindent.test.ts | 28 +++---- .../comments/browser/simpleCommentEditor.ts | 4 +- 19 files changed, 191 insertions(+), 112 deletions(-) diff --git a/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts b/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts index 69bf235e7cbeb..2946799ece828 100644 --- a/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts +++ b/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts @@ -60,6 +60,7 @@ import { INotificationService, Severity } from 'vs/platform/notification/common/ import { editorErrorForeground, editorHintForeground, editorInfoForeground, editorWarningForeground } from 'vs/platform/theme/common/colorRegistry'; import { IThemeService, registerThemingParticipant } from 'vs/platform/theme/common/themeService'; import { MenuId } from 'vs/platform/actions/common/actions'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeEditor { @@ -252,6 +253,7 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, + @ILanguageService private readonly languageService: ILanguageService, @ILanguageConfigurationService private readonly languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { @@ -1645,6 +1647,7 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE DOMLineBreaksComputerFactory.create(dom.getWindow(this._domElement)), MonospaceLineBreaksComputerFactory.create(this._configuration.options), (callback) => dom.scheduleAtNextAnimationFrame(dom.getWindow(this._domElement), callback), + this.languageService, this.languageConfigurationService, this._themeService, attachedView, diff --git a/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts b/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts index 9fb3a8e69c2b9..ad58286ba4f4b 100644 --- a/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts +++ b/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts @@ -8,6 +8,7 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser'; import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService'; import { CodeEditorWidget, ICodeEditorWidgetOptions } from 'vs/editor/browser/widget/codeEditor/codeEditorWidget'; import { ConfigurationChangedEvent, IEditorOptions } from 'vs/editor/common/config/editorOptions'; +import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures'; import { IAccessibilityService } from 'vs/platform/accessibility/common/accessibility'; @@ -33,10 +34,11 @@ export class EmbeddedCodeEditorWidget extends CodeEditorWidget { @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, + @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService ) { - super(domElement, { ...parentEditor.getRawOptions(), overflowWidgetsDomNode: parentEditor.getOverflowWidgetsDomNode() }, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); + super(domElement, { ...parentEditor.getRawOptions(), overflowWidgetsDomNode: parentEditor.getOverflowWidgetsDomNode() }, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); this._parentEditor = parentEditor; this._overwriteOptions = options; diff --git a/src/vs/editor/common/commands/shiftCommand.ts b/src/vs/editor/common/commands/shiftCommand.ts index 3c0c9bea6fb29..367178d990870 100644 --- a/src/vs/editor/common/commands/shiftCommand.ts +++ b/src/vs/editor/common/commands/shiftCommand.ts @@ -13,6 +13,7 @@ import { ITextModel } from 'vs/editor/common/model'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getEnterAction } from 'vs/editor/common/languages/enterAction'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IShiftCommandOpts { isUnshift: boolean; @@ -83,6 +84,7 @@ export class ShiftCommand implements ICommand { constructor( range: Selection, opts: IShiftCommandOpts, + @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { this._opts = opts; @@ -147,7 +149,7 @@ export class ShiftCommand implements ICommand { // The current line is "miss-aligned", so let's see if this is expected... // This can only happen when it has trailing commas in the indent if (model.tokenization.isCheapToTokenize(lineNumber - 1)) { - const enterAction = getEnterAction(this._opts.autoIndent, model, new Range(lineNumber - 1, model.getLineMaxColumn(lineNumber - 1), lineNumber - 1, model.getLineMaxColumn(lineNumber - 1)), this._languageConfigurationService); + const enterAction = getEnterAction(this._opts.autoIndent, model, new Range(lineNumber - 1, model.getLineMaxColumn(lineNumber - 1), lineNumber - 1, model.getLineMaxColumn(lineNumber - 1)), this._languageService, this._languageConfigurationService); if (enterAction) { extraSpaces = previousLineExtraSpaces; if (enterAction.appendText) { diff --git a/src/vs/editor/common/cursor/cursorTypeOperations.ts b/src/vs/editor/common/cursor/cursorTypeOperations.ts index ffa80cbb63ca7..db1d54e56bc69 100644 --- a/src/vs/editor/common/cursor/cursorTypeOperations.ts +++ b/src/vs/editor/common/cursor/cursorTypeOperations.ts @@ -40,7 +40,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageConfigurationService); + }, config.languageService, config.languageConfigurationService); } return commands; } @@ -55,7 +55,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageConfigurationService); + }, config.languageService, config.languageConfigurationService); } return commands; } @@ -155,7 +155,7 @@ export class TypeOperations { let action: IndentAction | EnterAction | null = null; let indentation: string = ''; - const expectedIndentAction = getInheritIndentForLine(config.autoIndent, model, lineNumber, false, config.languageConfigurationService); + const expectedIndentAction = getInheritIndentForLine(config.autoIndent, model, lineNumber, false, config.languageService, config.languageConfigurationService); if (expectedIndentAction) { action = expectedIndentAction.action; indentation = expectedIndentAction.indentation; @@ -175,7 +175,7 @@ export class TypeOperations { } const maxColumn = model.getLineMaxColumn(lastLineNumber); - const expectedEnterAction = getEnterAction(config.autoIndent, model, new Range(lastLineNumber, maxColumn, lastLineNumber, maxColumn), config.languageConfigurationService); + const expectedEnterAction = getEnterAction(config.autoIndent, model, new Range(lastLineNumber, maxColumn, lastLineNumber, maxColumn), config.languageService, config.languageConfigurationService); if (expectedEnterAction) { indentation = expectedEnterAction.indentation + expectedEnterAction.appendText; } @@ -255,7 +255,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageConfigurationService); + }, config.languageService, config.languageConfigurationService); } } return commands; @@ -306,7 +306,7 @@ export class TypeOperations { return TypeOperations._typeCommand(range, '\n' + config.normalizeIndentation(indentation), keepPosition); } - const r = getEnterAction(config.autoIndent, model, range, config.languageConfigurationService); + const r = getEnterAction(config.autoIndent, model, range, config.languageService, config.languageConfigurationService); if (r) { if (r.indentAction === IndentAction.None) { // Nothing special @@ -348,7 +348,7 @@ export class TypeOperations { normalizeIndentation: (indent) => { return config.normalizeIndentation(indent); } - }, config.languageConfigurationService); + }, config.languageService, config.languageConfigurationService); if (ir) { let oldEndViewColumn = config.visibleColumnFromColumn(model, range.getEndPosition()); @@ -402,7 +402,7 @@ export class TypeOperations { unshiftIndent: (indentation) => { return TypeOperations.unshiftIndent(config, indentation); }, - }, config.languageConfigurationService); + }, config.languageService, config.languageConfigurationService); if (actualIndentation === null) { return null; diff --git a/src/vs/editor/common/cursorCommon.ts b/src/vs/editor/common/cursorCommon.ts index c5411aa853987..48520e173aa4a 100644 --- a/src/vs/editor/common/cursorCommon.ts +++ b/src/vs/editor/common/cursorCommon.ts @@ -17,6 +17,7 @@ import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; import { IElectricAction } from 'vs/editor/common/languages/supports/electricCharacter'; import { CursorColumns } from 'vs/editor/common/core/cursorColumns'; import { normalizeIndentation } from 'vs/editor/common/core/indentation'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IColumnSelectData { isReal: boolean; @@ -106,6 +107,7 @@ export class CursorConfiguration { languageId: string, modelOptions: TextModelResolvedOptions, configuration: IEditorConfiguration, + public readonly languageService: ILanguageService, public readonly languageConfigurationService: ILanguageConfigurationService ) { this._languageId = languageId; diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 90d5e05c5da5f..d783073fec1f7 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -10,14 +10,15 @@ import { IndentAction } from 'vs/editor/common/languages/languageConfiguration'; import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; import { Position } from 'vs/editor/common/core/position'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IVirtualModel { tokenization: { - getLineTokens(lineNumber: number): LineTokens; + getLineTokens(lineNumber: number): IViewLineTokens; getLanguageId(): string; getLanguageIdAtPosition(lineNumber: number, column: number): string; }; @@ -77,6 +78,7 @@ export function getInheritIndentForLine( model: IVirtualModel, lineNumber: number, honorIntentialIndent: boolean = true, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): { indentation: string; action: IndentAction | null; line?: number } | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -87,7 +89,7 @@ export function getInheritIndentForLine( if (!indentRulesSupport) { return null; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageService, languageConfigurationService); if (lineNumber <= 1) { return { @@ -222,6 +224,7 @@ export function getGoodIndentForLine( languageId: string, lineNumber: number, indentConverter: IIndentConverter, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): string | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -238,8 +241,8 @@ export function getGoodIndentForLine( return null; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageConfigurationService); - const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageService, languageConfigurationService); + const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageService, languageConfigurationService); if (indent) { const inheritLine = indent.line; @@ -306,6 +309,7 @@ export function getIndentForEnter( model: ITextModel, range: Range, indentConverter: IIndentConverter, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): { beforeEnter: string; afterEnter: string } | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -318,27 +322,31 @@ export function getIndentForEnter( } model.tokenization.forceTokenization(range.startLineNumber); - const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const afterEnterText = processedContext.afterRangeText; - const beforeEnterText = processedContext.beforeRangeText; - const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterText); + const afterEnterProcessedData = processedContext.afterRangeProcessedData; + const beforeEnterProcessedData = processedContext.beforeRangeProcessedData; + const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedData.processedLine); const virtualModel: IVirtualModel = { tokenization: { - getLineTokens: (lineNumber: number) => { - return model.tokenization.getLineTokens(lineNumber); + getLineTokens: (lineNumber: number): IViewLineTokens => { + if (lineNumber === range.startLineNumber) { + return beforeEnterProcessedData.processedLineTokens; + } else { + return model.tokenization.getLineTokens(lineNumber); + } }, - getLanguageId: () => { + getLanguageId: (): string => { return model.getLanguageId(); }, - getLanguageIdAtPosition: (lineNumber: number, column: number) => { + getLanguageIdAtPosition: (lineNumber: number, column: number): string => { return model.getLanguageIdAtPosition(lineNumber, column); }, }, - getLineContent: (lineNumber: number) => { + getLineContent: (lineNumber: number): string => { if (lineNumber === range.startLineNumber) { - return beforeEnterText; + return beforeEnterProcessedData.processedLine; } else { return model.getLineContent(lineNumber); } @@ -348,7 +356,7 @@ export function getIndentForEnter( const embeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); const currentLine = model.getLineContent(range.startLineNumber); const currentLineIndent = strings.getLeadingWhitespace(currentLine); - const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); + const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageService, languageConfigurationService); if (!afterEnterAction) { const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; return { @@ -363,7 +371,7 @@ export function getIndentForEnter( afterEnterIndent = indentConverter.shiftIndent(afterEnterIndent); } - if (indentRulesSupport.shouldDecrease(afterEnterText)) { + if (indentRulesSupport.shouldDecrease(afterEnterProcessedData.processedLine)) { afterEnterIndent = indentConverter.unshiftIndent(afterEnterIndent); } @@ -383,6 +391,7 @@ export function getIndentActionForType( range: Range, ch: string, indentConverter: IIndentConverter, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): string | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -400,10 +409,10 @@ export function getIndentActionForType( return null; } - const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const beforeRangeText = processedContext.beforeRangeText; - const afterRangeText = processedContext.afterRangeText; + const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; + const afterRangeText = processedContext.afterRangeProcessedData.processedLine; const textAroundRange = beforeRangeText + afterRangeText; const textAroundRangeWithCharacter = beforeRangeText + ch + afterRangeText; @@ -412,7 +421,7 @@ export function getIndentActionForType( if (!indentRulesSupport.shouldDecrease(textAroundRange) && indentRulesSupport.shouldDecrease(textAroundRangeWithCharacter)) { // after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner. // 1. Get inherited indent action - const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageConfigurationService); + const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageService, languageConfigurationService); if (!r) { return null; } diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index 30a88d9d5bb53..49894a180c970 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -9,11 +9,13 @@ import { IndentAction, CompleteEnterAction } from 'vs/editor/common/languages/la import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getIndentationAtPosition, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export function getEnterAction( autoIndent: EditorAutoIndentStrategy, model: ITextModel, range: Range, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): CompleteEnterAction | null { model.tokenization.forceTokenization(range.startLineNumber); @@ -22,10 +24,13 @@ export function getEnterAction( if (!richEditSupport) { return null; } - const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); + const previousLineText = processedContext.previousLineProcessedData.processedLine; + const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; + const afterRangeText = processedContext.afterRangeProcessedData.processedLine; - const enterResult = richEditSupport.onEnter(autoIndent, processedContext.previousLineText, processedContext.beforeRangeText, processedContext.afterRangeText); + const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeRangeText, afterRangeText); if (!enterResult) { return null; } diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 0a3f74351d839..3e16d26ba2f99 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -12,6 +12,13 @@ import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageService } from 'vs/editor/common/languages/language'; +import { ILanguageIdCodec } from 'vs/editor/common/languages'; + +interface ProcessedLineData { + processedLine: string; + processedLineTokens: IViewLineTokens; +}; /** * This class is a wrapper class around {@link IndentRulesSupport}. @@ -26,10 +33,11 @@ export class ProcessedIndentRulesSupport { constructor( model: IVirtualModel, indentRulesSupport: IndentRulesSupport, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ) { this._indentRulesSupport = indentRulesSupport; - this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + this._indentationLineProcessor = new IndentationLineProcessor(model, languageService.languageIdCodec, languageConfigurationService); } /** @@ -76,44 +84,47 @@ export class ProcessedIndentRulesSupport { export class IndentationContextProcessor { private readonly model: ITextModel; + private readonly languageIdCodec: ILanguageIdCodec; private readonly indentationLineProcessor: IndentationLineProcessor; constructor( model: ITextModel, + languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ) { this.model = model; - this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + this.languageIdCodec = languageService.languageIdCodec; + this.indentationLineProcessor = new IndentationLineProcessor(model, this.languageIdCodec, languageConfigurationService); } /** * Returns the processed text, stripped from the language configuration brackets within the string, comment and regex tokens, around the given range */ getProcessedContextAroundRange(range: Range): { - beforeRangeText: string; - afterRangeText: string; - previousLineText: string; + beforeRangeProcessedData: ProcessedLineData; + afterRangeProcessedData: ProcessedLineData; + previousLineProcessedData: ProcessedLineData; } { this.model.tokenization.forceTokenization(range.startLineNumber); const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const beforeRangeText = this._getProcessedTextBeforeRange(range, scopedLineTokens); - const afterRangeText = this._getProcessedTextAfterRange(range, scopedLineTokens); - const previousLineText = this._getProcessedPreviousLine(range, scopedLineTokens); - return { beforeRangeText, afterRangeText, previousLineText }; + const beforeRangeProcessedData = this._getProcessedTextBeforeRange(range, scopedLineTokens); + const afterRangeProcessedData = this._getProcessedTextAfterRange(range, scopedLineTokens); + const previousLineProcessedData = this._getProcessedPreviousLine(range, scopedLineTokens); + return { beforeRangeProcessedData, afterRangeProcessedData, previousLineProcessedData }; } - private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): string { + private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const slicedTokensBefore = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokensBefore); - return processedLine; + const processedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(slicedTokensBefore); + return processedLineData; } - private _getProcessedTextAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): string { + private _getProcessedTextAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { let columnIndexWithinScope: number; let lineTokens: LineTokens; if (range.isEmpty()) { @@ -127,11 +138,11 @@ export class IndentationContextProcessor { const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length; const slicedTokensAfter = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLine = this.indentationLineProcessor.getProcessedLineForTokens(slicedTokensAfter); - return processedLine; + const processedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(slicedTokensAfter); + return processedLineData; } - private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): string { + private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { // Utility functions const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { @@ -154,20 +165,20 @@ export class IndentationContextProcessor { const previousLineNumber = range.startLineNumber - 1; const isFirstLine = previousLineNumber === 0; if (isFirstLine) { - return ''; + return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; } const canScopeExtendOnPreviousLine = scopedLineTokens.doesScopeStartAtOffsetZero(); if (!canScopeExtendOnPreviousLine) { - return ''; + return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; } const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; if (!doesLanguageContinueOnPreviousLine) { - return ''; + return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; } const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); - const processedPreviousScopedLine = this.indentationLineProcessor.getProcessedLineForTokens(previousSlicedLineTokens); - return processedPreviousScopedLine; + const processedPreviousScopedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(previousSlicedLineTokens); + return processedPreviousScopedLineData; } } @@ -179,6 +190,7 @@ class IndentationLineProcessor { constructor( private readonly model: IVirtualModel, + private readonly languageIdCodec: ILanguageIdCodec, private readonly languageConfigurationService: ILanguageConfigurationService ) { } @@ -197,7 +209,7 @@ class IndentationLineProcessor { // Main code const tokens = this.model.tokenization.getLineTokens(lineNumber); - let processedLine = this.getProcessedLineForTokens(tokens); + let processedLine = this.getProcessedLineAndTokens(tokens).processedLine; if (newIndentation !== undefined) { processedLine = adjustIndentation(processedLine, newIndentation); } @@ -207,7 +219,7 @@ class IndentationLineProcessor { /** * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. */ - getProcessedLineForTokens(tokens: IViewLineTokens): string { + getProcessedLineAndTokens(tokens: IViewLineTokens): ProcessedLineData { // Utility functions const isTokenTypeToProcess = (tokenType: StandardTokenType): boolean => { @@ -242,21 +254,32 @@ class IndentationLineProcessor { const languageId = tokens.getLanguageId(0); const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; if (!brackets) { - return tokens.getLineContent(); + return { processedLine: tokens.getLineContent(), processedLineTokens: tokens }; } const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); + let offset = 0; let processedLine = ''; + const processedTokensArray: number[] = []; tokens.forEach((tokenIndex: number) => { const tokenType = tokens.getStandardTokenType(tokenIndex); const text = tokens.getTokenText(tokenIndex); + const metadata = tokens.getMetadata(tokenIndex); + const endOffset = tokens.getEndOffset(tokenIndex) - offset; + processedTokensArray.push(endOffset); + processedTokensArray.push(metadata); + if (isTokenTypeToProcess(tokenType)) { - processedLine += removeBracketsFromText(text); + const processedText = removeBracketsFromText(text); + processedLine += processedText; + offset += text.length - processedText.length; } else { processedLine += text; } }); - return processedLine; + const processedTokens = new Uint32Array(processedTokensArray); + const processedLineTokens = new LineTokens(processedTokens, processedLine, this.languageIdCodec); + return { processedLine, processedLineTokens }; } } diff --git a/src/vs/editor/common/viewModel/viewModelImpl.ts b/src/vs/editor/common/viewModel/viewModelImpl.ts index c5af99cca77bf..1653e5ffcd187 100644 --- a/src/vs/editor/common/viewModel/viewModelImpl.ts +++ b/src/vs/editor/common/viewModel/viewModelImpl.ts @@ -40,6 +40,7 @@ import { FocusChangedEvent, HiddenAreasChangedEvent, ModelContentChangedEvent, M import { IViewModelLines, ViewModelLinesFromModelAsIs, ViewModelLinesFromProjectedModel } from 'vs/editor/common/viewModel/viewModelLines'; import { IThemeService } from 'vs/platform/theme/common/themeService'; import { GlyphMarginLanesModel } from 'vs/editor/common/viewModel/glyphLanesModel'; +import { ILanguageService } from 'vs/editor/common/languages/language'; const USE_IDENTITY_LINES_COLLECTION = true; @@ -68,6 +69,7 @@ export class ViewModel extends Disposable implements IViewModel { domLineBreaksComputerFactory: ILineBreaksComputerFactory, monospaceLineBreaksComputerFactory: ILineBreaksComputerFactory, scheduleAtNextAnimationFrame: (callback: () => void) => IDisposable, + private readonly languageService: ILanguageService, private readonly languageConfigurationService: ILanguageConfigurationService, private readonly _themeService: IThemeService, private readonly _attachedView: IAttachedView, @@ -79,7 +81,7 @@ export class ViewModel extends Disposable implements IViewModel { this.model = model; this._eventDispatcher = new ViewModelEventDispatcher(); this.onEvent = this._eventDispatcher.onEvent; - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); this._updateConfigurationViewLineCount = this._register(new RunOnceScheduler(() => this._updateConfigurationViewLineCountNow(), 0)); this._hasFocus = false; this._viewportStart = ViewportStart.create(this.model); @@ -271,7 +273,7 @@ export class ViewModel extends Disposable implements IViewModel { stableViewport.recoverViewportStart(this.coordinatesConverter, this.viewLayout); if (CursorConfiguration.shouldRecreate(e)) { - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); } } @@ -431,13 +433,13 @@ export class ViewModel extends Disposable implements IViewModel { this._register(this.model.onDidChangeLanguageConfiguration((e) => { this._eventDispatcher.emitSingleViewEvent(new viewEvents.ViewLanguageConfigurationEvent()); - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelLanguageConfigurationChangedEvent(e)); })); this._register(this.model.onDidChangeLanguage((e) => { - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelLanguageChangedEvent(e)); })); @@ -459,7 +461,7 @@ export class ViewModel extends Disposable implements IViewModel { this._updateConfigurationViewLineCount.schedule(); } - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelOptionsChangedEvent(e)); diff --git a/src/vs/editor/contrib/indentation/browser/indentation.ts b/src/vs/editor/contrib/indentation/browser/indentation.ts index 9860a3b677e50..63cb67dd97312 100644 --- a/src/vs/editor/contrib/indentation/browser/indentation.ts +++ b/src/vs/editor/contrib/indentation/browser/indentation.ts @@ -25,6 +25,7 @@ import * as nls from 'vs/nls'; import { IQuickInputService } from 'vs/platform/quickinput/common/quickInput'; import { getGoodIndentForLine, getIndentMetadata } from 'vs/editor/common/languages/autoIndent'; import { getReindentEditOperations } from '../common/indentation'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export class IndentationToSpacesAction extends EditorAction { public static readonly ID = 'editor.action.indentationToSpaces'; @@ -253,12 +254,13 @@ export class ReindentLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); + const languageService = accessor.get(ILanguageService); const model = editor.getModel(); if (!model) { return; } - const edits = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const edits = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); if (edits.length > 0) { editor.pushUndoStop(); editor.executeEdits(this.id, edits); @@ -282,6 +284,7 @@ export class ReindentSelectedLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); + const languageService = accessor.get(ILanguageService); const model = editor.getModel(); if (!model) { @@ -311,7 +314,7 @@ export class ReindentSelectedLinesAction extends EditorAction { startLineNumber--; } - const editOperations = getReindentEditOperations(model, languageConfigurationService, startLineNumber, endLineNumber); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, startLineNumber, endLineNumber); edits.push(...editOperations); } @@ -378,6 +381,7 @@ export class AutoIndentOnPaste implements IEditorContribution { constructor( private readonly editor: ICodeEditor, + @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { @@ -449,7 +453,7 @@ export class AutoIndentOnPaste implements IEditorContribution { let firstLineText = model.getLineContent(startLineNumber); if (!/\S/.test(firstLineText.substring(0, range.startColumn - 1))) { - const indentOfFirstLine = getGoodIndentForLine(autoIndent, model, model.getLanguageId(), startLineNumber, indentConverter, this._languageConfigurationService); + const indentOfFirstLine = getGoodIndentForLine(autoIndent, model, model.getLanguageId(), startLineNumber, indentConverter, this._languageService, this._languageConfigurationService); if (indentOfFirstLine !== null) { const oldIndentation = strings.getLeadingWhitespace(firstLineText); @@ -509,7 +513,7 @@ export class AutoIndentOnPaste implements IEditorContribution { } } }; - const indentOfSecondLine = getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageId(), startLineNumber + 1, indentConverter, this._languageConfigurationService); + const indentOfSecondLine = getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageId(), startLineNumber + 1, indentConverter, this._languageService, this._languageConfigurationService); if (indentOfSecondLine !== null) { const newSpaceCntOfSecondLine = indentUtils.getSpaceCnt(indentOfSecondLine, tabSize); const oldSpaceCntOfSecondLine = indentUtils.getSpaceCnt(strings.getLeadingWhitespace(model.getLineContent(startLineNumber + 1)), tabSize); diff --git a/src/vs/editor/contrib/indentation/common/indentation.ts b/src/vs/editor/contrib/indentation/common/indentation.ts index 41cc7cb8068b8..8a2c021956017 100644 --- a/src/vs/editor/contrib/indentation/common/indentation.ts +++ b/src/vs/editor/contrib/indentation/common/indentation.ts @@ -8,11 +8,12 @@ import { ShiftCommand } from 'vs/editor/common/commands/shiftCommand'; import { EditOperation, ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { normalizeIndentation } from 'vs/editor/common/core/indentation'; import { Selection } from 'vs/editor/common/core/selection'; +import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { ITextModel } from 'vs/editor/common/model'; -export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { +export function getReindentEditOperations(model: ITextModel, languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { if (model.getLineCount() === 1 && model.getLineMaxColumn(1) === 1) { // Model is empty return []; @@ -23,7 +24,7 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati return []; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageService, languageConfigurationService); endLineNumber = Math.min(endLineNumber, model.getLineCount()); // Skip `unIndentedLinePattern` lines diff --git a/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts b/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts index 45b3fafba7146..8e27d75a40198 100644 --- a/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts +++ b/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts @@ -26,6 +26,7 @@ import { MenuId } from 'vs/platform/actions/common/actions'; import { KeybindingWeight } from 'vs/platform/keybinding/common/keybindingsRegistry'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; +import { ILanguageService } from 'vs/editor/common/languages/language'; // copy lines @@ -174,13 +175,14 @@ abstract class AbstractMoveLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); + const languageService = accessor.get(ILanguageService); const commands: ICommand[] = []; const selections = editor.getSelections() || []; const autoIndent = editor.getOption(EditorOption.autoIndent); for (const selection of selections) { - commands.push(new MoveLinesCommand(selection, this.down, autoIndent, languageConfigurationService)); + commands.push(new MoveLinesCommand(selection, this.down, autoIndent, languageService, languageConfigurationService)); } editor.pushUndoStop(); diff --git a/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts b/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts index 68614a2f432f9..f5015acf881bd 100644 --- a/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts +++ b/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts @@ -16,6 +16,7 @@ import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import * as indentUtils from 'vs/editor/contrib/indentation/common/indentUtils'; import { getGoodIndentForLine, getIndentMetadata, IIndentConverter, IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { getEnterAction } from 'vs/editor/common/languages/enterAction'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export class MoveLinesCommand implements ICommand { @@ -31,6 +32,7 @@ export class MoveLinesCommand implements ICommand { selection: Selection, isMovingDown: boolean, autoIndent: EditorAutoIndentStrategy, + @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { this._selection = selection; @@ -133,6 +135,7 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(movingLineNumber, 1), s.startLineNumber, indentConverter, + this._languageService, this._languageConfigurationService ); if (indentOfMovingLine !== null) { @@ -175,6 +178,7 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(movingLineNumber, 1), s.startLineNumber + 1, indentConverter, + this._languageService, this._languageConfigurationService ); @@ -226,6 +230,7 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(s.startLineNumber, 1), movingLineNumber, indentConverter, + this._languageService, this._languageConfigurationService ); if (indentOfFirstLine !== null) { @@ -301,7 +306,7 @@ export class MoveLinesCommand implements ICommand { if (strings.lastNonWhitespaceIndex(futureAboveLineText) >= 0) { // break const maxColumn = model.getLineMaxColumn(futureAboveLineNumber); - const enter = getEnterAction(this._autoIndent, model, new Range(futureAboveLineNumber, maxColumn, futureAboveLineNumber, maxColumn), this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(futureAboveLineNumber, maxColumn, futureAboveLineNumber, maxColumn), this._languageService, this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } else { // go upwards, starting from `line - 1` @@ -322,7 +327,7 @@ export class MoveLinesCommand implements ICommand { } const maxColumn = model.getLineMaxColumn(validPrecedingLine); - const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageService, this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } } @@ -350,7 +355,7 @@ export class MoveLinesCommand implements ICommand { } const maxColumn = model.getLineMaxColumn(validPrecedingLine); - const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageService, this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } diff --git a/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts b/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts index 65941bd977bf7..b905c6628e61f 100644 --- a/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts +++ b/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts @@ -19,37 +19,43 @@ const enum MoveLinesDirection { Down } -function testMoveLinesDownCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownCommand(MoveLinesDirection.Down, lines, selection, expectedLines, expectedSelection, languageConfigurationService); +function testMoveLinesDownCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownCommand(MoveLinesDirection.Down, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); } -function testMoveLinesUpCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownCommand(MoveLinesDirection.Up, lines, selection, expectedLines, expectedSelection, languageConfigurationService); +function testMoveLinesUpCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownCommand(MoveLinesDirection.Up, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); } -function testMoveLinesDownWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Down, languageId, lines, selection, expectedLines, expectedSelection, languageConfigurationService); +function testMoveLinesDownWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Down, languageId, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); } -function testMoveLinesUpWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Up, languageId, lines, selection, expectedLines, expectedSelection, languageConfigurationService); +function testMoveLinesUpWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Up, languageId, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); } -function testMoveLinesUpOrDownCommand(direction: MoveLinesDirection, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService) { +function testMoveLinesUpOrDownCommand(direction: MoveLinesDirection, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService) { const disposables = new DisposableStore(); if (!languageConfigurationService) { languageConfigurationService = disposables.add(new TestLanguageConfigurationService()); } - testCommand(lines, null, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Advanced, languageConfigurationService), expectedLines, expectedSelection); + if (!languageService) { + languageService = disposables.add(new LanguageService()); + } + testCommand(lines, null, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Advanced, languageService, languageConfigurationService), expectedLines, expectedSelection); disposables.dispose(); } -function testMoveLinesUpOrDownWithIndentCommand(direction: MoveLinesDirection, languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService) { +function testMoveLinesUpOrDownWithIndentCommand(direction: MoveLinesDirection, languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService) { const disposables = new DisposableStore(); if (!languageConfigurationService) { languageConfigurationService = disposables.add(new TestLanguageConfigurationService()); } - testCommand(lines, languageId, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Full, languageConfigurationService), expectedLines, expectedSelection); + if (!languageService) { + languageService = disposables.add(new LanguageService()); + } + testCommand(lines, languageId, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Full, languageService, languageConfigurationService), expectedLines, expectedSelection); disposables.dispose(); } @@ -333,6 +339,7 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => { '}' ], new Selection(1, 1, 1, 1), + languageService, languageConfigurationService ); @@ -367,6 +374,7 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => { '];' ], new Selection(2, 5, 2, 5), + languageService, languageConfigurationService ); @@ -453,6 +461,7 @@ suite('Editor - contrib - Move Lines Command honors onEnter Rules', () => { '}' ], new Selection(4, 9, 6, 10), + languageService, languageConfigurationService ); diff --git a/src/vs/editor/standalone/browser/standaloneCodeEditor.ts b/src/vs/editor/standalone/browser/standaloneCodeEditor.ts index 84deb53c57d91..52025c47dca97 100644 --- a/src/vs/editor/standalone/browser/standaloneCodeEditor.ts +++ b/src/vs/editor/standalone/browser/standaloneCodeEditor.ts @@ -278,13 +278,14 @@ export class StandaloneCodeEditor extends CodeEditorWidget implements IStandalon @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, + @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { const options = { ..._options }; options.ariaLabel = options.ariaLabel || StandaloneCodeEditorNLS.editorViewAccessibleLabel; options.ariaLabel = options.ariaLabel + ';' + (StandaloneCodeEditorNLS.accessibilityHelpMessage); - super(domElement, options, {}, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); + super(domElement, options, {}, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); if (keybindingService instanceof StandaloneKeybindingService) { this._standaloneKeybindingService = keybindingService; @@ -440,7 +441,7 @@ export class StandaloneEditor extends StandaloneCodeEditor implements IStandalon } const _model: ITextModel | null | undefined = options.model; delete options.model; - super(domElement, options, instantiationService, codeEditorService, commandService, contextKeyService, hoverService, keybindingService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); + super(domElement, options, instantiationService, codeEditorService, commandService, contextKeyService, hoverService, keybindingService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); this._configurationService = configurationService; this._standaloneThemeService = themeService; diff --git a/src/vs/editor/test/browser/commands/shiftCommand.test.ts b/src/vs/editor/test/browser/commands/shiftCommand.test.ts index a769b21c4f9fc..9ad2fb2eb5262 100644 --- a/src/vs/editor/test/browser/commands/shiftCommand.test.ts +++ b/src/vs/editor/test/browser/commands/shiftCommand.test.ts @@ -13,6 +13,7 @@ import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { LanguageService } from 'vs/editor/common/services/languageService'; import { getEditOperation, testCommand } from 'vs/editor/test/browser/testCommand'; import { javascriptOnEnterRules } from 'vs/editor/test/common/modes/supports/onEnterRules'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; @@ -61,7 +62,7 @@ function testShiftCommand(lines: string[], languageId: string | null, useTabStop insertSpaces: false, useTabStops: useTabStops, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); } function testUnshiftCommand(lines: string[], languageId: string | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection, prepare?: (accessor: ServicesAccessor, disposables: DisposableStore) => void): void { @@ -72,7 +73,7 @@ function testUnshiftCommand(lines: string[], languageId: string | null, useTabSt insertSpaces: false, useTabStops: useTabStops, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); } function prepareDocBlockCommentLanguage(accessor: ServicesAccessor, disposables: DisposableStore) { @@ -686,7 +687,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -732,7 +733,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -778,7 +779,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: false, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -824,7 +825,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -859,7 +860,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: false, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), [ '\tHello world!', 'another line' @@ -963,6 +964,7 @@ suite('Editor Commands - ShiftCommand', () => { function _assertUnshiftCommand(tabSize: number, indentSize: number, insertSpaces: boolean, text: string[], expected: ISingleEditOperation[]): void { return withEditorModel(text, (model) => { + const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); const op = new ShiftCommand(new Selection(1, 1, text.length + 1, 1), { isUnshift: true, @@ -971,7 +973,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: insertSpaces, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, testLanguageConfigurationService); + }, languageService, testLanguageConfigurationService); const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); @@ -980,6 +982,7 @@ suite('Editor Commands - ShiftCommand', () => { function _assertShiftCommand(tabSize: number, indentSize: number, insertSpaces: boolean, text: string[], expected: ISingleEditOperation[]): void { return withEditorModel(text, (model) => { + const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); const op = new ShiftCommand(new Selection(1, 1, text.length + 1, 1), { isUnshift: false, @@ -988,7 +991,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: insertSpaces, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, testLanguageConfigurationService); + }, languageService, testLanguageConfigurationService); const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); diff --git a/src/vs/editor/test/browser/viewModel/testViewModel.ts b/src/vs/editor/test/browser/viewModel/testViewModel.ts index fe7f0c6e2dbf9..476816ffcddff 100644 --- a/src/vs/editor/test/browser/viewModel/testViewModel.ts +++ b/src/vs/editor/test/browser/viewModel/testViewModel.ts @@ -11,6 +11,7 @@ import { MonospaceLineBreaksComputerFactory } from 'vs/editor/common/viewModel/m import { createTextModel } from 'vs/editor/test/common/testTextModel'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService'; +import { LanguageService } from 'vs/editor/common/services/languageService'; export function testViewModel(text: string[], options: IEditorOptions, callback: (viewModel: ViewModel, model: TextModel) => void): void { const EDITOR_ID = 1; @@ -18,8 +19,9 @@ export function testViewModel(text: string[], options: IEditorOptions, callback: const configuration = new TestConfiguration(options); const model = createTextModel(text.join('\n')); const monospaceLineBreaksComputerFactory = MonospaceLineBreaksComputerFactory.create(configuration.options); + const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); - const viewModel = new ViewModel(EDITOR_ID, configuration, model, monospaceLineBreaksComputerFactory, monospaceLineBreaksComputerFactory, null!, testLanguageConfigurationService, new TestThemeService(), { + const viewModel = new ViewModel(EDITOR_ID, configuration, model, monospaceLineBreaksComputerFactory, monospaceLineBreaksComputerFactory, null!, languageService, testLanguageConfigurationService, new TestThemeService(), { setVisibleLines(visibleLines, stabilized) { }, }); diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index 9344f1a46a636..d4607096c3bd8 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -90,11 +90,13 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { let disposables: DisposableStore; let instantiationService: TestInstantiationService; let languageConfigurationService: ILanguageConfigurationService; + let languageService: ILanguageService; setup(() => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + languageService = instantiationService.get(ILanguageService); disposables.add(instantiationService); disposables.add(registerLanguage(instantiationService, languageId)); disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); @@ -145,7 +147,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { if (trimmedLineContent.length === 0) { continue; } - const editOperation = getReindentEditOperations(model, languageConfigurationService, line, line + 1); + const editOperation = getReindentEditOperations(model, languageService, languageConfigurationService, line, line + 1); /* NOTE: Uncomment in order to see actual incorrect indentation diff model.applyEdits(editOperation); @@ -220,7 +222,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -248,7 +250,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' ){}', ].join('\n'); let model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - let editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + let editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); let operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -265,7 +267,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '){}', ].join('\n'); model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -292,7 +294,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { `}` ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -340,7 +342,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -369,7 +371,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '};', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -388,7 +390,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { `}`, ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -406,7 +408,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); let model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - let editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + let editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); fileContents = [ @@ -415,7 +417,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -432,7 +434,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' n + 1;', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -447,7 +449,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' */', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -463,7 +465,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); }); diff --git a/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts b/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts index bad0ff2cab0e9..0610daceabd36 100644 --- a/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts +++ b/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts @@ -37,6 +37,7 @@ import { MessageController } from 'vs/editor/contrib/message/browser/messageCont import { SelectionClipboardContributionID } from 'vs/workbench/contrib/codeEditor/browser/selectionClipboard'; import { MenuId } from 'vs/platform/actions/common/actions'; import { HoverController } from 'vs/editor/contrib/hover/browser/hoverController'; +import { ILanguageService } from 'vs/editor/common/languages/language'; export const ctxCommentEditorFocused = new RawContextKey('commentEditorFocused', false); export const MIN_EDITOR_HEIGHT = 5 * 18; @@ -62,6 +63,7 @@ export class SimpleCommentEditor extends CodeEditorWidget { @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, + @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { @@ -87,7 +89,7 @@ export class SimpleCommentEditor extends CodeEditorWidget { contextMenuId: MenuId.SimpleEditorContext }; - super(domElement, options, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, scopedContextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); + super(domElement, options, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, scopedContextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); this._commentEditorFocused = ctxCommentEditorFocused.bindTo(scopedContextKeyService); this._commentEditorEmpty = CommentContextKeys.commentIsEmpty.bindTo(scopedContextKeyService); From 058f92c41b9744588679cbf4de057382f71ae907 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 11:56:28 +0200 Subject: [PATCH 36/47] disposing the language service in the tests --- src/vs/editor/test/browser/commands/shiftCommand.test.ts | 2 ++ src/vs/editor/test/browser/viewModel/testViewModel.ts | 1 + .../contrib/codeEditor/test/node/autoindent.test.ts | 5 ++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/vs/editor/test/browser/commands/shiftCommand.test.ts b/src/vs/editor/test/browser/commands/shiftCommand.test.ts index 9ad2fb2eb5262..3b95da368ecd2 100644 --- a/src/vs/editor/test/browser/commands/shiftCommand.test.ts +++ b/src/vs/editor/test/browser/commands/shiftCommand.test.ts @@ -977,6 +977,7 @@ suite('Editor Commands - ShiftCommand', () => { const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); + languageService.dispose(); }); } @@ -995,6 +996,7 @@ suite('Editor Commands - ShiftCommand', () => { const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); + languageService.dispose(); }); } }); diff --git a/src/vs/editor/test/browser/viewModel/testViewModel.ts b/src/vs/editor/test/browser/viewModel/testViewModel.ts index 476816ffcddff..122aedc6f95c4 100644 --- a/src/vs/editor/test/browser/viewModel/testViewModel.ts +++ b/src/vs/editor/test/browser/viewModel/testViewModel.ts @@ -32,4 +32,5 @@ export function testViewModel(text: string[], options: IEditorOptions, callback: model.dispose(); configuration.dispose(); testLanguageConfigurationService.dispose(); + languageService.dispose(); } diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index d4607096c3bd8..d29aaadd55f30 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -36,8 +36,7 @@ const enum LanguageId { TypeScript = 'ts-test' } -function registerLanguage(instantiationService: TestInstantiationService, languageId: LanguageId): IDisposable { - const languageService = instantiationService.get(ILanguageService) +function registerLanguage(languageService: ILanguageService, languageId: LanguageId): IDisposable { return languageService.registerLanguage({ id: languageId }); } @@ -98,7 +97,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { languageConfigurationService = instantiationService.get(ILanguageConfigurationService); languageService = instantiationService.get(ILanguageService); disposables.add(instantiationService); - disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerLanguage(languageService, languageId)); disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); }); From d5a9fb8f805adce4a8950de07d056742a07e7754 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 13:49:04 +0200 Subject: [PATCH 37/47] reverting the language service changes and adding the languageIdCodec into the IViewLineTokens interface --- .../widget/codeEditor/codeEditorWidget.ts | 3 -- .../codeEditor/embeddedCodeEditorWidget.ts | 4 +-- src/vs/editor/common/commands/shiftCommand.ts | 4 +-- .../common/cursor/cursorTypeOperations.ts | 16 ++++----- src/vs/editor/common/cursorCommon.ts | 2 -- src/vs/editor/common/languages/autoIndent.ts | 19 ++++------- src/vs/editor/common/languages/enterAction.ts | 4 +-- src/vs/editor/common/languages/supports.ts | 3 ++ .../supports/indentationLineProcessor.ts | 20 ++++------- .../editor/common/services/languageService.ts | 2 ++ .../common/services/languagesRegistry.ts | 2 ++ src/vs/editor/common/tokens/lineTokens.ts | 13 +++++--- .../editor/common/viewModel/viewModelImpl.ts | 12 +++---- .../indentation/browser/indentation.ts | 12 +++---- .../contrib/indentation/common/indentation.ts | 5 ++- .../browser/linesOperations.ts | 4 +-- .../browser/moveLinesCommand.ts | 11 ++----- .../test/browser/moveLinesCommand.test.ts | 33 +++++++------------ .../browser/standaloneCodeEditor.ts | 5 ++- .../browser/commands/shiftCommand.test.ts | 23 +++++-------- .../test/browser/viewModel/testViewModel.ts | 5 +-- .../editor/test/common/core/testLineToken.ts | 5 +++ .../codeEditor/test/node/autoindent.test.ts | 26 +++++++-------- .../comments/browser/simpleCommentEditor.ts | 4 +-- src/vs/workbench/test/common/utils.ts | 2 +- 25 files changed, 100 insertions(+), 139 deletions(-) diff --git a/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts b/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts index 2946799ece828..69bf235e7cbeb 100644 --- a/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts +++ b/src/vs/editor/browser/widget/codeEditor/codeEditorWidget.ts @@ -60,7 +60,6 @@ import { INotificationService, Severity } from 'vs/platform/notification/common/ import { editorErrorForeground, editorHintForeground, editorInfoForeground, editorWarningForeground } from 'vs/platform/theme/common/colorRegistry'; import { IThemeService, registerThemingParticipant } from 'vs/platform/theme/common/themeService'; import { MenuId } from 'vs/platform/actions/common/actions'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeEditor { @@ -253,7 +252,6 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, - @ILanguageService private readonly languageService: ILanguageService, @ILanguageConfigurationService private readonly languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { @@ -1647,7 +1645,6 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE DOMLineBreaksComputerFactory.create(dom.getWindow(this._domElement)), MonospaceLineBreaksComputerFactory.create(this._configuration.options), (callback) => dom.scheduleAtNextAnimationFrame(dom.getWindow(this._domElement), callback), - this.languageService, this.languageConfigurationService, this._themeService, attachedView, diff --git a/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts b/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts index ad58286ba4f4b..9fb3a8e69c2b9 100644 --- a/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts +++ b/src/vs/editor/browser/widget/codeEditor/embeddedCodeEditorWidget.ts @@ -8,7 +8,6 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser'; import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService'; import { CodeEditorWidget, ICodeEditorWidgetOptions } from 'vs/editor/browser/widget/codeEditor/codeEditorWidget'; import { ConfigurationChangedEvent, IEditorOptions } from 'vs/editor/common/config/editorOptions'; -import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures'; import { IAccessibilityService } from 'vs/platform/accessibility/common/accessibility'; @@ -34,11 +33,10 @@ export class EmbeddedCodeEditorWidget extends CodeEditorWidget { @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, - @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService ) { - super(domElement, { ...parentEditor.getRawOptions(), overflowWidgetsDomNode: parentEditor.getOverflowWidgetsDomNode() }, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); + super(domElement, { ...parentEditor.getRawOptions(), overflowWidgetsDomNode: parentEditor.getOverflowWidgetsDomNode() }, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); this._parentEditor = parentEditor; this._overwriteOptions = options; diff --git a/src/vs/editor/common/commands/shiftCommand.ts b/src/vs/editor/common/commands/shiftCommand.ts index 367178d990870..3c0c9bea6fb29 100644 --- a/src/vs/editor/common/commands/shiftCommand.ts +++ b/src/vs/editor/common/commands/shiftCommand.ts @@ -13,7 +13,6 @@ import { ITextModel } from 'vs/editor/common/model'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getEnterAction } from 'vs/editor/common/languages/enterAction'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IShiftCommandOpts { isUnshift: boolean; @@ -84,7 +83,6 @@ export class ShiftCommand implements ICommand { constructor( range: Selection, opts: IShiftCommandOpts, - @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { this._opts = opts; @@ -149,7 +147,7 @@ export class ShiftCommand implements ICommand { // The current line is "miss-aligned", so let's see if this is expected... // This can only happen when it has trailing commas in the indent if (model.tokenization.isCheapToTokenize(lineNumber - 1)) { - const enterAction = getEnterAction(this._opts.autoIndent, model, new Range(lineNumber - 1, model.getLineMaxColumn(lineNumber - 1), lineNumber - 1, model.getLineMaxColumn(lineNumber - 1)), this._languageService, this._languageConfigurationService); + const enterAction = getEnterAction(this._opts.autoIndent, model, new Range(lineNumber - 1, model.getLineMaxColumn(lineNumber - 1), lineNumber - 1, model.getLineMaxColumn(lineNumber - 1)), this._languageConfigurationService); if (enterAction) { extraSpaces = previousLineExtraSpaces; if (enterAction.appendText) { diff --git a/src/vs/editor/common/cursor/cursorTypeOperations.ts b/src/vs/editor/common/cursor/cursorTypeOperations.ts index db1d54e56bc69..ffa80cbb63ca7 100644 --- a/src/vs/editor/common/cursor/cursorTypeOperations.ts +++ b/src/vs/editor/common/cursor/cursorTypeOperations.ts @@ -40,7 +40,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageService, config.languageConfigurationService); + }, config.languageConfigurationService); } return commands; } @@ -55,7 +55,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageService, config.languageConfigurationService); + }, config.languageConfigurationService); } return commands; } @@ -155,7 +155,7 @@ export class TypeOperations { let action: IndentAction | EnterAction | null = null; let indentation: string = ''; - const expectedIndentAction = getInheritIndentForLine(config.autoIndent, model, lineNumber, false, config.languageService, config.languageConfigurationService); + const expectedIndentAction = getInheritIndentForLine(config.autoIndent, model, lineNumber, false, config.languageConfigurationService); if (expectedIndentAction) { action = expectedIndentAction.action; indentation = expectedIndentAction.indentation; @@ -175,7 +175,7 @@ export class TypeOperations { } const maxColumn = model.getLineMaxColumn(lastLineNumber); - const expectedEnterAction = getEnterAction(config.autoIndent, model, new Range(lastLineNumber, maxColumn, lastLineNumber, maxColumn), config.languageService, config.languageConfigurationService); + const expectedEnterAction = getEnterAction(config.autoIndent, model, new Range(lastLineNumber, maxColumn, lastLineNumber, maxColumn), config.languageConfigurationService); if (expectedEnterAction) { indentation = expectedEnterAction.indentation + expectedEnterAction.appendText; } @@ -255,7 +255,7 @@ export class TypeOperations { insertSpaces: config.insertSpaces, useTabStops: config.useTabStops, autoIndent: config.autoIndent - }, config.languageService, config.languageConfigurationService); + }, config.languageConfigurationService); } } return commands; @@ -306,7 +306,7 @@ export class TypeOperations { return TypeOperations._typeCommand(range, '\n' + config.normalizeIndentation(indentation), keepPosition); } - const r = getEnterAction(config.autoIndent, model, range, config.languageService, config.languageConfigurationService); + const r = getEnterAction(config.autoIndent, model, range, config.languageConfigurationService); if (r) { if (r.indentAction === IndentAction.None) { // Nothing special @@ -348,7 +348,7 @@ export class TypeOperations { normalizeIndentation: (indent) => { return config.normalizeIndentation(indent); } - }, config.languageService, config.languageConfigurationService); + }, config.languageConfigurationService); if (ir) { let oldEndViewColumn = config.visibleColumnFromColumn(model, range.getEndPosition()); @@ -402,7 +402,7 @@ export class TypeOperations { unshiftIndent: (indentation) => { return TypeOperations.unshiftIndent(config, indentation); }, - }, config.languageService, config.languageConfigurationService); + }, config.languageConfigurationService); if (actualIndentation === null) { return null; diff --git a/src/vs/editor/common/cursorCommon.ts b/src/vs/editor/common/cursorCommon.ts index 48520e173aa4a..c5411aa853987 100644 --- a/src/vs/editor/common/cursorCommon.ts +++ b/src/vs/editor/common/cursorCommon.ts @@ -17,7 +17,6 @@ import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; import { IElectricAction } from 'vs/editor/common/languages/supports/electricCharacter'; import { CursorColumns } from 'vs/editor/common/core/cursorColumns'; import { normalizeIndentation } from 'vs/editor/common/core/indentation'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IColumnSelectData { isReal: boolean; @@ -107,7 +106,6 @@ export class CursorConfiguration { languageId: string, modelOptions: TextModelResolvedOptions, configuration: IEditorConfiguration, - public readonly languageService: ILanguageService, public readonly languageConfigurationService: ILanguageConfigurationService ) { this._languageId = languageId; diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index d783073fec1f7..3552f78ea4233 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -14,7 +14,6 @@ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; import { Position } from 'vs/editor/common/core/position'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export interface IVirtualModel { tokenization: { @@ -78,7 +77,6 @@ export function getInheritIndentForLine( model: IVirtualModel, lineNumber: number, honorIntentialIndent: boolean = true, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): { indentation: string; action: IndentAction | null; line?: number } | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -89,7 +87,7 @@ export function getInheritIndentForLine( if (!indentRulesSupport) { return null; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageService, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageConfigurationService); if (lineNumber <= 1) { return { @@ -224,7 +222,6 @@ export function getGoodIndentForLine( languageId: string, lineNumber: number, indentConverter: IIndentConverter, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): string | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -241,8 +238,8 @@ export function getGoodIndentForLine( return null; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageService, languageConfigurationService); - const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageService, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageConfigurationService); + const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); if (indent) { const inheritLine = indent.line; @@ -309,7 +306,6 @@ export function getIndentForEnter( model: ITextModel, range: Range, indentConverter: IIndentConverter, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): { beforeEnter: string; afterEnter: string } | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -322,7 +318,7 @@ export function getIndentForEnter( } model.tokenization.forceTokenization(range.startLineNumber); - const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); const afterEnterProcessedData = processedContext.afterRangeProcessedData; const beforeEnterProcessedData = processedContext.beforeRangeProcessedData; @@ -356,7 +352,7 @@ export function getIndentForEnter( const embeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); const currentLine = model.getLineContent(range.startLineNumber); const currentLineIndent = strings.getLeadingWhitespace(currentLine); - const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageService, languageConfigurationService); + const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); if (!afterEnterAction) { const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; return { @@ -391,7 +387,6 @@ export function getIndentActionForType( range: Range, ch: string, indentConverter: IIndentConverter, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): string | null { if (autoIndent < EditorAutoIndentStrategy.Full) { @@ -409,7 +404,7 @@ export function getIndentActionForType( return null; } - const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; const afterRangeText = processedContext.afterRangeProcessedData.processedLine; @@ -421,7 +416,7 @@ export function getIndentActionForType( if (!indentRulesSupport.shouldDecrease(textAroundRange) && indentRulesSupport.shouldDecrease(textAroundRangeWithCharacter)) { // after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner. // 1. Get inherited indent action - const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageService, languageConfigurationService); + const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageConfigurationService); if (!r) { return null; } diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index 49894a180c970..ec8e3bf9b79cd 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -9,13 +9,11 @@ import { IndentAction, CompleteEnterAction } from 'vs/editor/common/languages/la import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { getIndentationAtPosition, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export function getEnterAction( autoIndent: EditorAutoIndentStrategy, model: ITextModel, range: Range, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ): CompleteEnterAction | null { model.tokenization.forceTokenization(range.startLineNumber); @@ -24,7 +22,7 @@ export function getEnterAction( if (!richEditSupport) { return null; } - const indentationContextProcessor = new IndentationContextProcessor(model, languageService, languageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); const previousLineText = processedContext.previousLineProcessedData.processedLine; const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index bf434a95a241b..11311c4af2645 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -5,6 +5,7 @@ import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageIdCodec } from 'vs/editor/common/languages'; export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens { const tokenCount = context.getCount(); @@ -34,6 +35,7 @@ export function createScopedLineTokens(context: LineTokens, offset: number): Sco export class ScopedLineTokens { _scopedLineTokensBrand: void = undefined; + public readonly languageIdCodec: ILanguageIdCodec; public readonly languageId: string; private readonly _actual: LineTokens; private readonly _firstTokenIndex: number; @@ -55,6 +57,7 @@ export class ScopedLineTokens { this._lastTokenIndex = lastTokenIndex; this.firstCharOffset = firstCharOffset; this._lastCharOffset = lastCharOffset; + this.languageIdCodec = actual.languageIdCodec; } public getLineContent(): string { diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 3e16d26ba2f99..c245e85d885d4 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -12,8 +12,6 @@ import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; -import { ILanguageService } from 'vs/editor/common/languages/language'; -import { ILanguageIdCodec } from 'vs/editor/common/languages'; interface ProcessedLineData { processedLine: string; @@ -33,11 +31,10 @@ export class ProcessedIndentRulesSupport { constructor( model: IVirtualModel, indentRulesSupport: IndentRulesSupport, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ) { this._indentRulesSupport = indentRulesSupport; - this._indentationLineProcessor = new IndentationLineProcessor(model, languageService.languageIdCodec, languageConfigurationService); + this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } /** @@ -84,17 +81,14 @@ export class ProcessedIndentRulesSupport { export class IndentationContextProcessor { private readonly model: ITextModel; - private readonly languageIdCodec: ILanguageIdCodec; private readonly indentationLineProcessor: IndentationLineProcessor; constructor( model: ITextModel, - languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService ) { this.model = model; - this.languageIdCodec = languageService.languageIdCodec; - this.indentationLineProcessor = new IndentationLineProcessor(model, this.languageIdCodec, languageConfigurationService); + this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); } /** @@ -162,19 +156,20 @@ export class IndentationContextProcessor { } // Main code + const nullProcessedData: ProcessedLineData = { processedLine: '', processedLineTokens: LineTokens.createEmpty('', scopedLineTokens.languageIdCodec) }; const previousLineNumber = range.startLineNumber - 1; const isFirstLine = previousLineNumber === 0; if (isFirstLine) { - return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; + return nullProcessedData; } const canScopeExtendOnPreviousLine = scopedLineTokens.doesScopeStartAtOffsetZero(); if (!canScopeExtendOnPreviousLine) { - return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; + return nullProcessedData; } const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; if (!doesLanguageContinueOnPreviousLine) { - return { processedLine: '', processedLineTokens: LineTokens.createEmpty('', this.languageIdCodec) }; + return nullProcessedData; } const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); const processedPreviousScopedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(previousSlicedLineTokens); @@ -190,7 +185,6 @@ class IndentationLineProcessor { constructor( private readonly model: IVirtualModel, - private readonly languageIdCodec: ILanguageIdCodec, private readonly languageConfigurationService: ILanguageConfigurationService ) { } @@ -279,7 +273,7 @@ class IndentationLineProcessor { } }); const processedTokens = new Uint32Array(processedTokensArray); - const processedLineTokens = new LineTokens(processedTokens, processedLine, this.languageIdCodec); + const processedLineTokens = new LineTokens(processedTokens, processedLine, tokens.languageIdCodec); return { processedLine, processedLineTokens }; } } diff --git a/src/vs/editor/common/services/languageService.ts b/src/vs/editor/common/services/languageService.ts index 6d96f2a2502f1..c47292adb582e 100644 --- a/src/vs/editor/common/services/languageService.ts +++ b/src/vs/editor/common/services/languageService.ts @@ -35,6 +35,7 @@ export class LanguageService extends Disposable implements ILanguageService { constructor(warnOnOverwrite = false) { super(); LanguageService.instanceCount++; + console.log('inside of constructor of languages registry'); this._registry = this._register(new LanguagesRegistry(true, warnOnOverwrite)); this.languageIdCodec = this._registry.languageIdCodec; this._register(this._registry.onDidChange(() => this._onDidChange.fire())); @@ -42,6 +43,7 @@ export class LanguageService extends Disposable implements ILanguageService { public override dispose(): void { LanguageService.instanceCount--; + this._registry.dispose(); super.dispose(); } diff --git a/src/vs/editor/common/services/languagesRegistry.ts b/src/vs/editor/common/services/languagesRegistry.ts index 9216a625543ab..5db11f81bf3fc 100644 --- a/src/vs/editor/common/services/languagesRegistry.ts +++ b/src/vs/editor/common/services/languagesRegistry.ts @@ -80,6 +80,7 @@ export class LanguagesRegistry extends Disposable { constructor(useModesRegistry = true, warnOnOverwrite = false) { super(); + console.log('inside of constructor of LanguagesRegistry'); LanguagesRegistry.instanceCount++; this._warnOnOverwrite = warnOnOverwrite; @@ -99,6 +100,7 @@ export class LanguagesRegistry extends Disposable { } override dispose() { + console.log('inside of dispose of LanguagesRegistry') LanguagesRegistry.instanceCount--; super.dispose(); } diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index b17be218ca2c8..64031843baa5e 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -7,6 +7,7 @@ import { ILanguageIdCodec } from 'vs/editor/common/languages'; import { FontStyle, ColorId, StandardTokenType, MetadataConsts, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; export interface IViewLineTokens { + languageIdCodec: ILanguageIdCodec; equals(other: IViewLineTokens): boolean; getCount(): number; getStandardTokenType(tokenIndex: number): StandardTokenType; @@ -29,7 +30,8 @@ export class LineTokens implements IViewLineTokens { private readonly _tokens: Uint32Array; private readonly _tokensCount: number; private readonly _text: string; - private readonly _languageIdCodec: ILanguageIdCodec; + + public readonly languageIdCodec: ILanguageIdCodec; public static defaultTokenMetadata = ( (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) @@ -51,7 +53,7 @@ export class LineTokens implements IViewLineTokens { this._tokens = tokens; this._tokensCount = (this._tokens.length >>> 1); this._text = text; - this._languageIdCodec = decoder; + this.languageIdCodec = decoder; } public equals(other: IViewLineTokens): boolean { @@ -101,7 +103,7 @@ export class LineTokens implements IViewLineTokens { public getLanguageId(tokenIndex: number): string { const metadata = this._tokens[(tokenIndex << 1) + 1]; const languageId = TokenMetadata.getLanguageId(metadata); - return this._languageIdCodec.decodeLanguageId(languageId); + return this.languageIdCodec.decodeLanguageId(languageId); } public getStandardTokenType(tokenIndex: number): StandardTokenType { @@ -228,7 +230,7 @@ export class LineTokens implements IViewLineTokens { } } - return new LineTokens(new Uint32Array(newTokens), text, this._languageIdCodec); + return new LineTokens(new Uint32Array(newTokens), text, this.languageIdCodec); } public getTokenText(tokenIndex: number): string { @@ -256,12 +258,15 @@ class SliceLineTokens implements IViewLineTokens { private readonly _firstTokenIndex: number; private readonly _tokensCount: number; + public languageIdCodec: ILanguageIdCodec; + constructor(source: LineTokens, startOffset: number, endOffset: number, deltaOffset: number) { this._source = source; this._startOffset = startOffset; this._endOffset = endOffset; this._deltaOffset = deltaOffset; this._firstTokenIndex = source.findTokenIndexAtOffset(startOffset); + this.languageIdCodec = source.languageIdCodec; this._tokensCount = 0; for (let i = this._firstTokenIndex, len = source.getCount(); i < len; i++) { diff --git a/src/vs/editor/common/viewModel/viewModelImpl.ts b/src/vs/editor/common/viewModel/viewModelImpl.ts index 1653e5ffcd187..c5af99cca77bf 100644 --- a/src/vs/editor/common/viewModel/viewModelImpl.ts +++ b/src/vs/editor/common/viewModel/viewModelImpl.ts @@ -40,7 +40,6 @@ import { FocusChangedEvent, HiddenAreasChangedEvent, ModelContentChangedEvent, M import { IViewModelLines, ViewModelLinesFromModelAsIs, ViewModelLinesFromProjectedModel } from 'vs/editor/common/viewModel/viewModelLines'; import { IThemeService } from 'vs/platform/theme/common/themeService'; import { GlyphMarginLanesModel } from 'vs/editor/common/viewModel/glyphLanesModel'; -import { ILanguageService } from 'vs/editor/common/languages/language'; const USE_IDENTITY_LINES_COLLECTION = true; @@ -69,7 +68,6 @@ export class ViewModel extends Disposable implements IViewModel { domLineBreaksComputerFactory: ILineBreaksComputerFactory, monospaceLineBreaksComputerFactory: ILineBreaksComputerFactory, scheduleAtNextAnimationFrame: (callback: () => void) => IDisposable, - private readonly languageService: ILanguageService, private readonly languageConfigurationService: ILanguageConfigurationService, private readonly _themeService: IThemeService, private readonly _attachedView: IAttachedView, @@ -81,7 +79,7 @@ export class ViewModel extends Disposable implements IViewModel { this.model = model; this._eventDispatcher = new ViewModelEventDispatcher(); this.onEvent = this._eventDispatcher.onEvent; - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); this._updateConfigurationViewLineCount = this._register(new RunOnceScheduler(() => this._updateConfigurationViewLineCountNow(), 0)); this._hasFocus = false; this._viewportStart = ViewportStart.create(this.model); @@ -273,7 +271,7 @@ export class ViewModel extends Disposable implements IViewModel { stableViewport.recoverViewportStart(this.coordinatesConverter, this.viewLayout); if (CursorConfiguration.shouldRecreate(e)) { - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); } } @@ -433,13 +431,13 @@ export class ViewModel extends Disposable implements IViewModel { this._register(this.model.onDidChangeLanguageConfiguration((e) => { this._eventDispatcher.emitSingleViewEvent(new viewEvents.ViewLanguageConfigurationEvent()); - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelLanguageConfigurationChangedEvent(e)); })); this._register(this.model.onDidChangeLanguage((e) => { - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelLanguageChangedEvent(e)); })); @@ -461,7 +459,7 @@ export class ViewModel extends Disposable implements IViewModel { this._updateConfigurationViewLineCount.schedule(); } - this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageService, this.languageConfigurationService); + this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration, this.languageConfigurationService); this._cursor.updateConfiguration(this.cursorConfig); this._eventDispatcher.emitOutgoingEvent(new ModelOptionsChangedEvent(e)); diff --git a/src/vs/editor/contrib/indentation/browser/indentation.ts b/src/vs/editor/contrib/indentation/browser/indentation.ts index 63cb67dd97312..9860a3b677e50 100644 --- a/src/vs/editor/contrib/indentation/browser/indentation.ts +++ b/src/vs/editor/contrib/indentation/browser/indentation.ts @@ -25,7 +25,6 @@ import * as nls from 'vs/nls'; import { IQuickInputService } from 'vs/platform/quickinput/common/quickInput'; import { getGoodIndentForLine, getIndentMetadata } from 'vs/editor/common/languages/autoIndent'; import { getReindentEditOperations } from '../common/indentation'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export class IndentationToSpacesAction extends EditorAction { public static readonly ID = 'editor.action.indentationToSpaces'; @@ -254,13 +253,12 @@ export class ReindentLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); - const languageService = accessor.get(ILanguageService); const model = editor.getModel(); if (!model) { return; } - const edits = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const edits = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); if (edits.length > 0) { editor.pushUndoStop(); editor.executeEdits(this.id, edits); @@ -284,7 +282,6 @@ export class ReindentSelectedLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); - const languageService = accessor.get(ILanguageService); const model = editor.getModel(); if (!model) { @@ -314,7 +311,7 @@ export class ReindentSelectedLinesAction extends EditorAction { startLineNumber--; } - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, startLineNumber, endLineNumber); + const editOperations = getReindentEditOperations(model, languageConfigurationService, startLineNumber, endLineNumber); edits.push(...editOperations); } @@ -381,7 +378,6 @@ export class AutoIndentOnPaste implements IEditorContribution { constructor( private readonly editor: ICodeEditor, - @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { @@ -453,7 +449,7 @@ export class AutoIndentOnPaste implements IEditorContribution { let firstLineText = model.getLineContent(startLineNumber); if (!/\S/.test(firstLineText.substring(0, range.startColumn - 1))) { - const indentOfFirstLine = getGoodIndentForLine(autoIndent, model, model.getLanguageId(), startLineNumber, indentConverter, this._languageService, this._languageConfigurationService); + const indentOfFirstLine = getGoodIndentForLine(autoIndent, model, model.getLanguageId(), startLineNumber, indentConverter, this._languageConfigurationService); if (indentOfFirstLine !== null) { const oldIndentation = strings.getLeadingWhitespace(firstLineText); @@ -513,7 +509,7 @@ export class AutoIndentOnPaste implements IEditorContribution { } } }; - const indentOfSecondLine = getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageId(), startLineNumber + 1, indentConverter, this._languageService, this._languageConfigurationService); + const indentOfSecondLine = getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageId(), startLineNumber + 1, indentConverter, this._languageConfigurationService); if (indentOfSecondLine !== null) { const newSpaceCntOfSecondLine = indentUtils.getSpaceCnt(indentOfSecondLine, tabSize); const oldSpaceCntOfSecondLine = indentUtils.getSpaceCnt(strings.getLeadingWhitespace(model.getLineContent(startLineNumber + 1)), tabSize); diff --git a/src/vs/editor/contrib/indentation/common/indentation.ts b/src/vs/editor/contrib/indentation/common/indentation.ts index 8a2c021956017..41cc7cb8068b8 100644 --- a/src/vs/editor/contrib/indentation/common/indentation.ts +++ b/src/vs/editor/contrib/indentation/common/indentation.ts @@ -8,12 +8,11 @@ import { ShiftCommand } from 'vs/editor/common/commands/shiftCommand'; import { EditOperation, ISingleEditOperation } from 'vs/editor/common/core/editOperation'; import { normalizeIndentation } from 'vs/editor/common/core/indentation'; import { Selection } from 'vs/editor/common/core/selection'; -import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { ITextModel } from 'vs/editor/common/model'; -export function getReindentEditOperations(model: ITextModel, languageService: ILanguageService, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { +export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { if (model.getLineCount() === 1 && model.getLineMaxColumn(1) === 1) { // Model is empty return []; @@ -24,7 +23,7 @@ export function getReindentEditOperations(model: ITextModel, languageService: IL return []; } - const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageService, languageConfigurationService); + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); endLineNumber = Math.min(endLineNumber, model.getLineCount()); // Skip `unIndentedLinePattern` lines diff --git a/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts b/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts index 8e27d75a40198..45b3fafba7146 100644 --- a/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts +++ b/src/vs/editor/contrib/linesOperations/browser/linesOperations.ts @@ -26,7 +26,6 @@ import { MenuId } from 'vs/platform/actions/common/actions'; import { KeybindingWeight } from 'vs/platform/keybinding/common/keybindingsRegistry'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; -import { ILanguageService } from 'vs/editor/common/languages/language'; // copy lines @@ -175,14 +174,13 @@ abstract class AbstractMoveLinesAction extends EditorAction { public run(accessor: ServicesAccessor, editor: ICodeEditor): void { const languageConfigurationService = accessor.get(ILanguageConfigurationService); - const languageService = accessor.get(ILanguageService); const commands: ICommand[] = []; const selections = editor.getSelections() || []; const autoIndent = editor.getOption(EditorOption.autoIndent); for (const selection of selections) { - commands.push(new MoveLinesCommand(selection, this.down, autoIndent, languageService, languageConfigurationService)); + commands.push(new MoveLinesCommand(selection, this.down, autoIndent, languageConfigurationService)); } editor.pushUndoStop(); diff --git a/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts b/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts index f5015acf881bd..68614a2f432f9 100644 --- a/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts +++ b/src/vs/editor/contrib/linesOperations/browser/moveLinesCommand.ts @@ -16,7 +16,6 @@ import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import * as indentUtils from 'vs/editor/contrib/indentation/common/indentUtils'; import { getGoodIndentForLine, getIndentMetadata, IIndentConverter, IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { getEnterAction } from 'vs/editor/common/languages/enterAction'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export class MoveLinesCommand implements ICommand { @@ -32,7 +31,6 @@ export class MoveLinesCommand implements ICommand { selection: Selection, isMovingDown: boolean, autoIndent: EditorAutoIndentStrategy, - @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService ) { this._selection = selection; @@ -135,7 +133,6 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(movingLineNumber, 1), s.startLineNumber, indentConverter, - this._languageService, this._languageConfigurationService ); if (indentOfMovingLine !== null) { @@ -178,7 +175,6 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(movingLineNumber, 1), s.startLineNumber + 1, indentConverter, - this._languageService, this._languageConfigurationService ); @@ -230,7 +226,6 @@ export class MoveLinesCommand implements ICommand { model.getLanguageIdAtPosition(s.startLineNumber, 1), movingLineNumber, indentConverter, - this._languageService, this._languageConfigurationService ); if (indentOfFirstLine !== null) { @@ -306,7 +301,7 @@ export class MoveLinesCommand implements ICommand { if (strings.lastNonWhitespaceIndex(futureAboveLineText) >= 0) { // break const maxColumn = model.getLineMaxColumn(futureAboveLineNumber); - const enter = getEnterAction(this._autoIndent, model, new Range(futureAboveLineNumber, maxColumn, futureAboveLineNumber, maxColumn), this._languageService, this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(futureAboveLineNumber, maxColumn, futureAboveLineNumber, maxColumn), this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } else { // go upwards, starting from `line - 1` @@ -327,7 +322,7 @@ export class MoveLinesCommand implements ICommand { } const maxColumn = model.getLineMaxColumn(validPrecedingLine); - const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageService, this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } } @@ -355,7 +350,7 @@ export class MoveLinesCommand implements ICommand { } const maxColumn = model.getLineMaxColumn(validPrecedingLine); - const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageService, this._languageConfigurationService); + const enter = getEnterAction(this._autoIndent, model, new Range(validPrecedingLine, maxColumn, validPrecedingLine, maxColumn), this._languageConfigurationService); return this.parseEnterResult(model, indentConverter, tabSize, line, enter); } diff --git a/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts b/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts index b905c6628e61f..65941bd977bf7 100644 --- a/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts +++ b/src/vs/editor/contrib/linesOperations/test/browser/moveLinesCommand.test.ts @@ -19,43 +19,37 @@ const enum MoveLinesDirection { Down } -function testMoveLinesDownCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownCommand(MoveLinesDirection.Down, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); +function testMoveLinesDownCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownCommand(MoveLinesDirection.Down, lines, selection, expectedLines, expectedSelection, languageConfigurationService); } -function testMoveLinesUpCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownCommand(MoveLinesDirection.Up, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); +function testMoveLinesUpCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownCommand(MoveLinesDirection.Up, lines, selection, expectedLines, expectedSelection, languageConfigurationService); } -function testMoveLinesDownWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Down, languageId, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); +function testMoveLinesDownWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Down, languageId, lines, selection, expectedLines, expectedSelection, languageConfigurationService); } -function testMoveLinesUpWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService): void { - testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Up, languageId, lines, selection, expectedLines, expectedSelection, languageService, languageConfigurationService); +function testMoveLinesUpWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService): void { + testMoveLinesUpOrDownWithIndentCommand(MoveLinesDirection.Up, languageId, lines, selection, expectedLines, expectedSelection, languageConfigurationService); } -function testMoveLinesUpOrDownCommand(direction: MoveLinesDirection, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService) { +function testMoveLinesUpOrDownCommand(direction: MoveLinesDirection, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService) { const disposables = new DisposableStore(); if (!languageConfigurationService) { languageConfigurationService = disposables.add(new TestLanguageConfigurationService()); } - if (!languageService) { - languageService = disposables.add(new LanguageService()); - } - testCommand(lines, null, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Advanced, languageService, languageConfigurationService), expectedLines, expectedSelection); + testCommand(lines, null, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Advanced, languageConfigurationService), expectedLines, expectedSelection); disposables.dispose(); } -function testMoveLinesUpOrDownWithIndentCommand(direction: MoveLinesDirection, languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageService?: ILanguageService, languageConfigurationService?: ILanguageConfigurationService) { +function testMoveLinesUpOrDownWithIndentCommand(direction: MoveLinesDirection, languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection, languageConfigurationService?: ILanguageConfigurationService) { const disposables = new DisposableStore(); if (!languageConfigurationService) { languageConfigurationService = disposables.add(new TestLanguageConfigurationService()); } - if (!languageService) { - languageService = disposables.add(new LanguageService()); - } - testCommand(lines, languageId, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Full, languageService, languageConfigurationService), expectedLines, expectedSelection); + testCommand(lines, languageId, selection, (accessor, sel) => new MoveLinesCommand(sel, direction === MoveLinesDirection.Up ? false : true, EditorAutoIndentStrategy.Full, languageConfigurationService), expectedLines, expectedSelection); disposables.dispose(); } @@ -339,7 +333,6 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => { '}' ], new Selection(1, 1, 1, 1), - languageService, languageConfigurationService ); @@ -374,7 +367,6 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => { '];' ], new Selection(2, 5, 2, 5), - languageService, languageConfigurationService ); @@ -461,7 +453,6 @@ suite('Editor - contrib - Move Lines Command honors onEnter Rules', () => { '}' ], new Selection(4, 9, 6, 10), - languageService, languageConfigurationService ); diff --git a/src/vs/editor/standalone/browser/standaloneCodeEditor.ts b/src/vs/editor/standalone/browser/standaloneCodeEditor.ts index 52025c47dca97..84deb53c57d91 100644 --- a/src/vs/editor/standalone/browser/standaloneCodeEditor.ts +++ b/src/vs/editor/standalone/browser/standaloneCodeEditor.ts @@ -278,14 +278,13 @@ export class StandaloneCodeEditor extends CodeEditorWidget implements IStandalon @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, - @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { const options = { ..._options }; options.ariaLabel = options.ariaLabel || StandaloneCodeEditorNLS.editorViewAccessibleLabel; options.ariaLabel = options.ariaLabel + ';' + (StandaloneCodeEditorNLS.accessibilityHelpMessage); - super(domElement, options, {}, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); + super(domElement, options, {}, instantiationService, codeEditorService, commandService, contextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); if (keybindingService instanceof StandaloneKeybindingService) { this._standaloneKeybindingService = keybindingService; @@ -441,7 +440,7 @@ export class StandaloneEditor extends StandaloneCodeEditor implements IStandalon } const _model: ITextModel | null | undefined = options.model; delete options.model; - super(domElement, options, instantiationService, codeEditorService, commandService, contextKeyService, hoverService, keybindingService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); + super(domElement, options, instantiationService, codeEditorService, commandService, contextKeyService, hoverService, keybindingService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); this._configurationService = configurationService; this._standaloneThemeService = themeService; diff --git a/src/vs/editor/test/browser/commands/shiftCommand.test.ts b/src/vs/editor/test/browser/commands/shiftCommand.test.ts index 3b95da368ecd2..a769b21c4f9fc 100644 --- a/src/vs/editor/test/browser/commands/shiftCommand.test.ts +++ b/src/vs/editor/test/browser/commands/shiftCommand.test.ts @@ -13,7 +13,6 @@ import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { LanguageService } from 'vs/editor/common/services/languageService'; import { getEditOperation, testCommand } from 'vs/editor/test/browser/testCommand'; import { javascriptOnEnterRules } from 'vs/editor/test/common/modes/supports/onEnterRules'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; @@ -62,7 +61,7 @@ function testShiftCommand(lines: string[], languageId: string | null, useTabStop insertSpaces: false, useTabStops: useTabStops, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); + }, accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); } function testUnshiftCommand(lines: string[], languageId: string | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection, prepare?: (accessor: ServicesAccessor, disposables: DisposableStore) => void): void { @@ -73,7 +72,7 @@ function testUnshiftCommand(lines: string[], languageId: string | null, useTabSt insertSpaces: false, useTabStops: useTabStops, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); + }, accessor.get(ILanguageConfigurationService)), expectedLines, expectedSelection, undefined, prepare); } function prepareDocBlockCommentLanguage(accessor: ServicesAccessor, disposables: DisposableStore) { @@ -687,7 +686,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -733,7 +732,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -779,7 +778,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: false, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -825,7 +824,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: true, useTabStops: false, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageConfigurationService)), [ ' Written | Numeric', ' one | 1', @@ -860,7 +859,7 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: false, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, accessor.get(ILanguageService), accessor.get(ILanguageConfigurationService)), + }, accessor.get(ILanguageConfigurationService)), [ '\tHello world!', 'another line' @@ -964,7 +963,6 @@ suite('Editor Commands - ShiftCommand', () => { function _assertUnshiftCommand(tabSize: number, indentSize: number, insertSpaces: boolean, text: string[], expected: ISingleEditOperation[]): void { return withEditorModel(text, (model) => { - const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); const op = new ShiftCommand(new Selection(1, 1, text.length + 1, 1), { isUnshift: true, @@ -973,17 +971,15 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: insertSpaces, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, languageService, testLanguageConfigurationService); + }, testLanguageConfigurationService); const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); - languageService.dispose(); }); } function _assertShiftCommand(tabSize: number, indentSize: number, insertSpaces: boolean, text: string[], expected: ISingleEditOperation[]): void { return withEditorModel(text, (model) => { - const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); const op = new ShiftCommand(new Selection(1, 1, text.length + 1, 1), { isUnshift: false, @@ -992,11 +988,10 @@ suite('Editor Commands - ShiftCommand', () => { insertSpaces: insertSpaces, useTabStops: true, autoIndent: EditorAutoIndentStrategy.Full, - }, languageService, testLanguageConfigurationService); + }, testLanguageConfigurationService); const actual = getEditOperation(model, op); assert.deepStrictEqual(actual, expected); testLanguageConfigurationService.dispose(); - languageService.dispose(); }); } }); diff --git a/src/vs/editor/test/browser/viewModel/testViewModel.ts b/src/vs/editor/test/browser/viewModel/testViewModel.ts index 122aedc6f95c4..fe7f0c6e2dbf9 100644 --- a/src/vs/editor/test/browser/viewModel/testViewModel.ts +++ b/src/vs/editor/test/browser/viewModel/testViewModel.ts @@ -11,7 +11,6 @@ import { MonospaceLineBreaksComputerFactory } from 'vs/editor/common/viewModel/m import { createTextModel } from 'vs/editor/test/common/testTextModel'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService'; -import { LanguageService } from 'vs/editor/common/services/languageService'; export function testViewModel(text: string[], options: IEditorOptions, callback: (viewModel: ViewModel, model: TextModel) => void): void { const EDITOR_ID = 1; @@ -19,9 +18,8 @@ export function testViewModel(text: string[], options: IEditorOptions, callback: const configuration = new TestConfiguration(options); const model = createTextModel(text.join('\n')); const monospaceLineBreaksComputerFactory = MonospaceLineBreaksComputerFactory.create(configuration.options); - const languageService = new LanguageService(); const testLanguageConfigurationService = new TestLanguageConfigurationService(); - const viewModel = new ViewModel(EDITOR_ID, configuration, model, monospaceLineBreaksComputerFactory, monospaceLineBreaksComputerFactory, null!, languageService, testLanguageConfigurationService, new TestThemeService(), { + const viewModel = new ViewModel(EDITOR_ID, configuration, model, monospaceLineBreaksComputerFactory, monospaceLineBreaksComputerFactory, null!, testLanguageConfigurationService, new TestThemeService(), { setVisibleLines(visibleLines, stabilized) { }, }); @@ -32,5 +30,4 @@ export function testViewModel(text: string[], options: IEditorOptions, callback: model.dispose(); configuration.dispose(); testLanguageConfigurationService.dispose(); - languageService.dispose(); } diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index c10558800de5a..3a1f47665d3ec 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -5,6 +5,7 @@ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; import { ColorId, TokenMetadata, ITokenPresentation, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageIdCodec } from 'vs/editor/common/languages'; /** * A token on a line. @@ -72,6 +73,10 @@ export class TestLineTokens implements IViewLineTokens { this._actual = actual; } + public get languageIdCodec(): ILanguageIdCodec { + throw new Error('Not implemented'); + } + public equals(other: IViewLineTokens): boolean { if (other instanceof TestLineTokens) { return TestLineToken.equalsArr(this._actual, other._actual); diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index d29aaadd55f30..db2a09c22d8a5 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -146,7 +146,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { if (trimmedLineContent.length === 0) { continue; } - const editOperation = getReindentEditOperations(model, languageService, languageConfigurationService, line, line + 1); + const editOperation = getReindentEditOperations(model, languageConfigurationService, line, line + 1); /* NOTE: Uncomment in order to see actual incorrect indentation diff model.applyEdits(editOperation); @@ -221,7 +221,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -249,7 +249,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' ){}', ].join('\n'); let model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - let editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + let editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); let operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -266,7 +266,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '){}', ].join('\n'); model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -293,7 +293,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { `}` ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -341,7 +341,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); model.tokenization.forceTokenization(1); model.tokenization.forceTokenization(2); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 1); const operation = editOperations[0]; assert.deepStrictEqual(getIRange(operation.range), { @@ -370,7 +370,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '};', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -389,7 +389,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { `}`, ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -407,7 +407,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); let model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - let editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + let editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); fileContents = [ @@ -416,7 +416,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -433,7 +433,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' n + 1;', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -448,7 +448,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { ' */', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); @@ -464,7 +464,7 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { '}', ].join('\n'); const model = disposables.add(instantiateTextModel(instantiationService, fileContents, languageId, options)); - const editOperations = getReindentEditOperations(model, languageService, languageConfigurationService, 1, model.getLineCount()); + const editOperations = getReindentEditOperations(model, languageConfigurationService, 1, model.getLineCount()); assert.deepStrictEqual(editOperations.length, 0); }); }); diff --git a/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts b/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts index 0610daceabd36..bad0ff2cab0e9 100644 --- a/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts +++ b/src/vs/workbench/contrib/comments/browser/simpleCommentEditor.ts @@ -37,7 +37,6 @@ import { MessageController } from 'vs/editor/contrib/message/browser/messageCont import { SelectionClipboardContributionID } from 'vs/workbench/contrib/codeEditor/browser/selectionClipboard'; import { MenuId } from 'vs/platform/actions/common/actions'; import { HoverController } from 'vs/editor/contrib/hover/browser/hoverController'; -import { ILanguageService } from 'vs/editor/common/languages/language'; export const ctxCommentEditorFocused = new RawContextKey('commentEditorFocused', false); export const MIN_EDITOR_HEIGHT = 5 * 18; @@ -63,7 +62,6 @@ export class SimpleCommentEditor extends CodeEditorWidget { @IThemeService themeService: IThemeService, @INotificationService notificationService: INotificationService, @IAccessibilityService accessibilityService: IAccessibilityService, - @ILanguageService languageService: ILanguageService, @ILanguageConfigurationService languageConfigurationService: ILanguageConfigurationService, @ILanguageFeaturesService languageFeaturesService: ILanguageFeaturesService, ) { @@ -89,7 +87,7 @@ export class SimpleCommentEditor extends CodeEditorWidget { contextMenuId: MenuId.SimpleEditorContext }; - super(domElement, options, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, scopedContextKeyService, themeService, notificationService, accessibilityService, languageService, languageConfigurationService, languageFeaturesService); + super(domElement, options, codeEditorWidgetOptions, instantiationService, codeEditorService, commandService, scopedContextKeyService, themeService, notificationService, accessibilityService, languageConfigurationService, languageFeaturesService); this._commentEditorFocused = ctxCommentEditorFocused.bindTo(scopedContextKeyService); this._commentEditorEmpty = CommentContextKeys.commentIsEmpty.bindTo(scopedContextKeyService); diff --git a/src/vs/workbench/test/common/utils.ts b/src/vs/workbench/test/common/utils.ts index b28eb1d1c8bea..4330090956b56 100644 --- a/src/vs/workbench/test/common/utils.ts +++ b/src/vs/workbench/test/common/utils.ts @@ -18,5 +18,5 @@ export function assertCleanState(): void { // If this test fails, it is a clear indication that // your test or suite is leaking services (e.g. via leaking text models) // assert.strictEqual(LanguageService.instanceCount, 0, 'No leaking ILanguageService'); - assert.strictEqual(LanguagesRegistry.instanceCount, 0, 'Error: Test run should not leak in LanguagesRegistry.'); + assert.strictEqual(LanguagesRegistry.instanceCount, 0, `Error: Test run should not leak in LanguagesRegistry with count ${LanguagesRegistry.instanceCount}`); } From 33534aa9b908c799db789afab24c3034e6d97219 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 13:54:54 +0200 Subject: [PATCH 38/47] polishing further the code --- .../common/languages/supports/indentationLineProcessor.ts | 8 ++++---- src/vs/editor/common/services/languageService.ts | 2 -- src/vs/editor/common/services/languagesRegistry.ts | 2 -- src/vs/editor/common/tokens/lineTokens.ts | 2 +- src/vs/editor/test/common/core/testLineToken.ts | 8 ++++---- .../contrib/codeEditor/test/node/autoindent.test.ts | 7 +++---- src/vs/workbench/test/common/utils.ts | 2 +- 7 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index c245e85d885d4..0df196c906af8 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -156,20 +156,20 @@ export class IndentationContextProcessor { } // Main code - const nullProcessedData: ProcessedLineData = { processedLine: '', processedLineTokens: LineTokens.createEmpty('', scopedLineTokens.languageIdCodec) }; + const emptyProcessedData: ProcessedLineData = { processedLine: '', processedLineTokens: LineTokens.createEmpty('', scopedLineTokens.languageIdCodec) }; const previousLineNumber = range.startLineNumber - 1; const isFirstLine = previousLineNumber === 0; if (isFirstLine) { - return nullProcessedData; + return emptyProcessedData; } const canScopeExtendOnPreviousLine = scopedLineTokens.doesScopeStartAtOffsetZero(); if (!canScopeExtendOnPreviousLine) { - return nullProcessedData; + return emptyProcessedData; } const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; if (!doesLanguageContinueOnPreviousLine) { - return nullProcessedData; + return emptyProcessedData; } const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); const processedPreviousScopedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(previousSlicedLineTokens); diff --git a/src/vs/editor/common/services/languageService.ts b/src/vs/editor/common/services/languageService.ts index c47292adb582e..6d96f2a2502f1 100644 --- a/src/vs/editor/common/services/languageService.ts +++ b/src/vs/editor/common/services/languageService.ts @@ -35,7 +35,6 @@ export class LanguageService extends Disposable implements ILanguageService { constructor(warnOnOverwrite = false) { super(); LanguageService.instanceCount++; - console.log('inside of constructor of languages registry'); this._registry = this._register(new LanguagesRegistry(true, warnOnOverwrite)); this.languageIdCodec = this._registry.languageIdCodec; this._register(this._registry.onDidChange(() => this._onDidChange.fire())); @@ -43,7 +42,6 @@ export class LanguageService extends Disposable implements ILanguageService { public override dispose(): void { LanguageService.instanceCount--; - this._registry.dispose(); super.dispose(); } diff --git a/src/vs/editor/common/services/languagesRegistry.ts b/src/vs/editor/common/services/languagesRegistry.ts index 5db11f81bf3fc..9216a625543ab 100644 --- a/src/vs/editor/common/services/languagesRegistry.ts +++ b/src/vs/editor/common/services/languagesRegistry.ts @@ -80,7 +80,6 @@ export class LanguagesRegistry extends Disposable { constructor(useModesRegistry = true, warnOnOverwrite = false) { super(); - console.log('inside of constructor of LanguagesRegistry'); LanguagesRegistry.instanceCount++; this._warnOnOverwrite = warnOnOverwrite; @@ -100,7 +99,6 @@ export class LanguagesRegistry extends Disposable { } override dispose() { - console.log('inside of dispose of LanguagesRegistry') LanguagesRegistry.instanceCount--; super.dispose(); } diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 64031843baa5e..12b77bcbab878 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -258,7 +258,7 @@ class SliceLineTokens implements IViewLineTokens { private readonly _firstTokenIndex: number; private readonly _tokensCount: number; - public languageIdCodec: ILanguageIdCodec; + public readonly languageIdCodec: ILanguageIdCodec; constructor(source: LineTokens, startOffset: number, endOffset: number, deltaOffset: number) { this._source = source; diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index 3a1f47665d3ec..f3c4980794121 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -73,10 +73,6 @@ export class TestLineTokens implements IViewLineTokens { this._actual = actual; } - public get languageIdCodec(): ILanguageIdCodec { - throw new Error('Not implemented'); - } - public equals(other: IViewLineTokens): boolean { if (other instanceof TestLineTokens) { return TestLineToken.equalsArr(this._actual, other._actual); @@ -135,6 +131,10 @@ export class TestLineTokens implements IViewLineTokens { public forEach(callback: (tokenIndex: number) => void): void { throw new Error('Not implemented'); } + + public get languageIdCodec(): ILanguageIdCodec { + throw new Error('Not implemented'); + } } export class TestLineTokenFactory { diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index db2a09c22d8a5..9344f1a46a636 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -36,7 +36,8 @@ const enum LanguageId { TypeScript = 'ts-test' } -function registerLanguage(languageService: ILanguageService, languageId: LanguageId): IDisposable { +function registerLanguage(instantiationService: TestInstantiationService, languageId: LanguageId): IDisposable { + const languageService = instantiationService.get(ILanguageService) return languageService.registerLanguage({ id: languageId }); } @@ -89,15 +90,13 @@ suite('Auto-Reindentation - TypeScript/JavaScript', () => { let disposables: DisposableStore; let instantiationService: TestInstantiationService; let languageConfigurationService: ILanguageConfigurationService; - let languageService: ILanguageService; setup(() => { disposables = new DisposableStore(); instantiationService = createModelServices(disposables); languageConfigurationService = instantiationService.get(ILanguageConfigurationService); - languageService = instantiationService.get(ILanguageService); disposables.add(instantiationService); - disposables.add(registerLanguage(languageService, languageId)); + disposables.add(registerLanguage(instantiationService, languageId)); disposables.add(registerLanguageConfiguration(languageConfigurationService, languageId)); }); diff --git a/src/vs/workbench/test/common/utils.ts b/src/vs/workbench/test/common/utils.ts index 4330090956b56..b28eb1d1c8bea 100644 --- a/src/vs/workbench/test/common/utils.ts +++ b/src/vs/workbench/test/common/utils.ts @@ -18,5 +18,5 @@ export function assertCleanState(): void { // If this test fails, it is a clear indication that // your test or suite is leaking services (e.g. via leaking text models) // assert.strictEqual(LanguageService.instanceCount, 0, 'No leaking ILanguageService'); - assert.strictEqual(LanguagesRegistry.instanceCount, 0, `Error: Test run should not leak in LanguagesRegistry with count ${LanguagesRegistry.instanceCount}`); + assert.strictEqual(LanguagesRegistry.instanceCount, 0, 'Error: Test run should not leak in LanguagesRegistry.'); } From 4ee9475c817b28a465c6c76e976794a11b1ee393 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 16:25:14 +0200 Subject: [PATCH 39/47] using tokens directly and not using the method on the scopedLineTokens class --- src/vs/editor/common/languages/autoIndent.ts | 30 +++++----- src/vs/editor/common/languages/enterAction.ts | 8 +-- src/vs/editor/common/languages/supports.ts | 4 -- .../supports/indentationLineProcessor.ts | 57 +++++++++---------- 4 files changed, 45 insertions(+), 54 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 3552f78ea4233..a8da8beb0c837 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -319,16 +319,16 @@ export function getIndentForEnter( model.tokenization.forceTokenization(range.startLineNumber); const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); - const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const afterEnterProcessedData = processedContext.afterRangeProcessedData; - const beforeEnterProcessedData = processedContext.beforeRangeProcessedData; - const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedData.processedLine); + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const afterEnterProcessedTokens = processedContextTokens.afterRangeProcessedTokens; + const beforeEnterProcessedTokens = processedContextTokens.beforeRangeProcessedTokens; + const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedTokens.getLineContent()); const virtualModel: IVirtualModel = { tokenization: { getLineTokens: (lineNumber: number): IViewLineTokens => { if (lineNumber === range.startLineNumber) { - return beforeEnterProcessedData.processedLineTokens; + return beforeEnterProcessedTokens; } else { return model.tokenization.getLineTokens(lineNumber); } @@ -342,14 +342,14 @@ export function getIndentForEnter( }, getLineContent: (lineNumber: number): string => { if (lineNumber === range.startLineNumber) { - return beforeEnterProcessedData.processedLine; + return beforeEnterProcessedTokens.getLineContent(); } else { return model.getLineContent(lineNumber); } } }; - const embeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); + const embeddedLanguage = isLanguageDifferentFromLineStart(model, range.getStartPosition()); const currentLine = model.getLineContent(range.startLineNumber); const currentLineIndent = strings.getLeadingWhitespace(currentLine); const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); @@ -367,7 +367,7 @@ export function getIndentForEnter( afterEnterIndent = indentConverter.shiftIndent(afterEnterIndent); } - if (indentRulesSupport.shouldDecrease(afterEnterProcessedData.processedLine)) { + if (indentRulesSupport.shouldDecrease(afterEnterProcessedTokens.getLineContent())) { afterEnterIndent = indentConverter.unshiftIndent(afterEnterIndent); } @@ -392,8 +392,8 @@ export function getIndentActionForType( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } - const isPositionWithinEmbeddedLanguage = isWithinEmbeddedLanguage(model, range.getStartPosition()); - if (isPositionWithinEmbeddedLanguage) { + const languageIsDifferentFromLineStart = isLanguageDifferentFromLineStart(model, range.getStartPosition()); + if (languageIsDifferentFromLineStart) { // this line has mixed languages and indentation rules will not work return null; } @@ -405,9 +405,9 @@ export function getIndentActionForType( } const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); - const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; - const afterRangeText = processedContext.afterRangeProcessedData.processedLine; + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const beforeRangeText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); + const afterRangeText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); const textAroundRange = beforeRangeText + afterRangeText; const textAroundRangeWithCharacter = beforeRangeText + ch + afterRangeText; @@ -447,10 +447,10 @@ export function getIndentMetadata( return indentRulesSupport.getIndentMetadata(model.getLineContent(lineNumber)); } -export function isWithinEmbeddedLanguage(model: ITextModel, position: Position): boolean { +function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { const lineTokens = model.tokenization.getLineTokens(position.lineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); - const doesScopeStartAtOffsetZero = scopedLineTokens.doesScopeStartAtOffsetZero(); + const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; const isWithinEmbeddedLanguage = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; return isWithinEmbeddedLanguage; diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index ec8e3bf9b79cd..be290b7b090f3 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -23,10 +23,10 @@ export function getEnterAction( return null; } const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); - const processedContext = indentationContextProcessor.getProcessedContextAroundRange(range); - const previousLineText = processedContext.previousLineProcessedData.processedLine; - const beforeRangeText = processedContext.beforeRangeProcessedData.processedLine; - const afterRangeText = processedContext.afterRangeProcessedData.processedLine; + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const previousLineText = processedContextTokens.previousLineProcessedTokens.getLineContent(); + const beforeRangeText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); + const afterRangeText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeRangeText, afterRangeText); if (!enterResult) { diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 11311c4af2645..39eeaff67e97e 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -81,10 +81,6 @@ export class ScopedLineTokens { public getStandardTokenType(tokenIndex: number): StandardTokenType { return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } - - public doesScopeStartAtOffsetZero(): boolean { - return this.firstCharOffset === 0; - } } const enum IgnoreBracketsInTokens { diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 0df196c906af8..c95cbd2722532 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -13,11 +13,6 @@ import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens' import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; -interface ProcessedLineData { - processedLine: string; - processedLineTokens: IViewLineTokens; -}; - /** * This class is a wrapper class around {@link IndentRulesSupport}. * It processes the lines by removing the language configuration brackets from the regex, string and comment tokens. @@ -94,31 +89,31 @@ export class IndentationContextProcessor { /** * Returns the processed text, stripped from the language configuration brackets within the string, comment and regex tokens, around the given range */ - getProcessedContextAroundRange(range: Range): { - beforeRangeProcessedData: ProcessedLineData; - afterRangeProcessedData: ProcessedLineData; - previousLineProcessedData: ProcessedLineData; + getProcessedTokenContextAroundRange(range: Range): { + beforeRangeProcessedTokens: IViewLineTokens; + afterRangeProcessedTokens: IViewLineTokens; + previousLineProcessedTokens: IViewLineTokens; } { this.model.tokenization.forceTokenization(range.startLineNumber); const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const beforeRangeProcessedData = this._getProcessedTextBeforeRange(range, scopedLineTokens); - const afterRangeProcessedData = this._getProcessedTextAfterRange(range, scopedLineTokens); - const previousLineProcessedData = this._getProcessedPreviousLine(range, scopedLineTokens); - return { beforeRangeProcessedData, afterRangeProcessedData, previousLineProcessedData }; + const beforeRangeProcessedTokens = this._getProcessedTokensBeforeRange(range, scopedLineTokens); + const afterRangeProcessedTokens = this._getProcessedTokensAfterRange(range, scopedLineTokens); + const previousLineProcessedTokens = this._getProcessedPreviousLineTokens(range, scopedLineTokens); + return { beforeRangeProcessedTokens, afterRangeProcessedTokens, previousLineProcessedTokens }; } - private _getProcessedTextBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { + private _getProcessedTokensBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset; const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const slicedTokensBefore = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(slicedTokensBefore); - return processedLineData; + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokensBefore); + return processedTokens; } - private _getProcessedTextAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { + private _getProcessedTokensAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { let columnIndexWithinScope: number; let lineTokens: LineTokens; if (range.isEmpty()) { @@ -132,11 +127,11 @@ export class IndentationContextProcessor { const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length; const slicedTokensAfter = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(slicedTokensAfter); - return processedLineData; + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokensAfter); + return processedTokens; } - private _getProcessedPreviousLine(range: Range, scopedLineTokens: ScopedLineTokens): ProcessedLineData { + private _getProcessedPreviousLineTokens(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { // Utility functions const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { @@ -156,24 +151,24 @@ export class IndentationContextProcessor { } // Main code - const emptyProcessedData: ProcessedLineData = { processedLine: '', processedLineTokens: LineTokens.createEmpty('', scopedLineTokens.languageIdCodec) }; + const emptyTokens = LineTokens.createEmpty('', scopedLineTokens.languageIdCodec); const previousLineNumber = range.startLineNumber - 1; const isFirstLine = previousLineNumber === 0; if (isFirstLine) { - return emptyProcessedData; + return emptyTokens; } - const canScopeExtendOnPreviousLine = scopedLineTokens.doesScopeStartAtOffsetZero(); + const canScopeExtendOnPreviousLine = scopedLineTokens.firstCharOffset === 0; if (!canScopeExtendOnPreviousLine) { - return emptyProcessedData; + return emptyTokens; } const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; if (!doesLanguageContinueOnPreviousLine) { - return emptyProcessedData; + return emptyTokens; } const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); - const processedPreviousScopedLineData = this.indentationLineProcessor.getProcessedLineAndTokens(previousSlicedLineTokens); - return processedPreviousScopedLineData; + const processedTokens = this.indentationLineProcessor.getProcessedTokens(previousSlicedLineTokens); + return processedTokens; } } @@ -203,7 +198,7 @@ class IndentationLineProcessor { // Main code const tokens = this.model.tokenization.getLineTokens(lineNumber); - let processedLine = this.getProcessedLineAndTokens(tokens).processedLine; + let processedLine = this.getProcessedTokens(tokens).getLineContent(); if (newIndentation !== undefined) { processedLine = adjustIndentation(processedLine, newIndentation); } @@ -213,7 +208,7 @@ class IndentationLineProcessor { /** * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. */ - getProcessedLineAndTokens(tokens: IViewLineTokens): ProcessedLineData { + getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { // Utility functions const isTokenTypeToProcess = (tokenType: StandardTokenType): boolean => { @@ -248,7 +243,7 @@ class IndentationLineProcessor { const languageId = tokens.getLanguageId(0); const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; if (!brackets) { - return { processedLine: tokens.getLineContent(), processedLineTokens: tokens }; + return tokens; } const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); @@ -274,6 +269,6 @@ class IndentationLineProcessor { }); const processedTokens = new Uint32Array(processedTokensArray); const processedLineTokens = new LineTokens(processedTokens, processedLine, tokens.languageIdCodec); - return { processedLine, processedLineTokens }; + return processedLineTokens; } } From 353b7f7b013f52436736d1e1671d22cd0911ce34 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 16:29:37 +0200 Subject: [PATCH 40/47] extracting the virtual model creation into a separate function --- src/vs/editor/common/languages/autoIndent.ts | 56 +++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index a8da8beb0c837..99888647363ff 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -324,31 +324,7 @@ export function getIndentForEnter( const beforeEnterProcessedTokens = processedContextTokens.beforeRangeProcessedTokens; const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedTokens.getLineContent()); - const virtualModel: IVirtualModel = { - tokenization: { - getLineTokens: (lineNumber: number): IViewLineTokens => { - if (lineNumber === range.startLineNumber) { - return beforeEnterProcessedTokens; - } else { - return model.tokenization.getLineTokens(lineNumber); - } - }, - getLanguageId: (): string => { - return model.getLanguageId(); - }, - getLanguageIdAtPosition: (lineNumber: number, column: number): string => { - return model.getLanguageIdAtPosition(lineNumber, column); - }, - }, - getLineContent: (lineNumber: number): string => { - if (lineNumber === range.startLineNumber) { - return beforeEnterProcessedTokens.getLineContent(); - } else { - return model.getLineContent(lineNumber); - } - } - }; - + const virtualModel = createVirtualModelWithModifiedTokensAtLine(model, range.startLineNumber, beforeEnterProcessedTokens); const embeddedLanguage = isLanguageDifferentFromLineStart(model, range.getStartPosition()); const currentLine = model.getLineContent(range.startLineNumber); const currentLineIndent = strings.getLeadingWhitespace(currentLine); @@ -447,6 +423,34 @@ export function getIndentMetadata( return indentRulesSupport.getIndentMetadata(model.getLineContent(lineNumber)); } +function createVirtualModelWithModifiedTokensAtLine(model: ITextModel, modifiedLineNumber: number, modifiedTokens: IViewLineTokens): IVirtualModel { + const virtualModel: IVirtualModel = { + tokenization: { + getLineTokens: (lineNumber: number): IViewLineTokens => { + if (lineNumber === modifiedLineNumber) { + return modifiedTokens; + } else { + return model.tokenization.getLineTokens(lineNumber); + } + }, + getLanguageId: (): string => { + return model.getLanguageId(); + }, + getLanguageIdAtPosition: (lineNumber: number, column: number): string => { + return model.getLanguageIdAtPosition(lineNumber, column); + }, + }, + getLineContent: (lineNumber: number): string => { + if (lineNumber === modifiedLineNumber) { + return modifiedTokens.getLineContent(); + } else { + return model.getLineContent(lineNumber); + } + } + }; + return virtualModel; +} + function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { const lineTokens = model.tokenization.getLineTokens(position.lineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); @@ -454,4 +458,4 @@ function isLanguageDifferentFromLineStart(model: ITextModel, position: Position) const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; const isWithinEmbeddedLanguage = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; return isWithinEmbeddedLanguage; -}; +} From 33d93f08cc16f52454c68e3d2307c2d5607fbd82 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 16:40:26 +0200 Subject: [PATCH 41/47] creating a static constructor directly on the line tokens --- .../supports/indentationLineProcessor.ts | 17 ++++++----------- src/vs/editor/common/tokens/lineTokens.ts | 12 ++++++++++++ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index c95cbd2722532..bd2ffbd37c4b7 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -211,7 +211,7 @@ class IndentationLineProcessor { getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { // Utility functions - const isTokenTypeToProcess = (tokenType: StandardTokenType): boolean => { + const isTokenTypeFromWhichToRemoveBrackets = (tokenType: StandardTokenType): boolean => { return tokenType === StandardTokenType.String || tokenType === StandardTokenType.RegEx || tokenType === StandardTokenType.Comment; @@ -248,27 +248,22 @@ class IndentationLineProcessor { const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - let offset = 0; let processedLine = ''; - const processedTokensArray: number[] = []; + const textAndMetadata: { text: string, metadata: number }[] = []; tokens.forEach((tokenIndex: number) => { const tokenType = tokens.getStandardTokenType(tokenIndex); const text = tokens.getTokenText(tokenIndex); const metadata = tokens.getMetadata(tokenIndex); - const endOffset = tokens.getEndOffset(tokenIndex) - offset; - processedTokensArray.push(endOffset); - processedTokensArray.push(metadata); - - if (isTokenTypeToProcess(tokenType)) { + if (isTokenTypeFromWhichToRemoveBrackets(tokenType)) { const processedText = removeBracketsFromText(text); processedLine += processedText; - offset += text.length - processedText.length; + textAndMetadata.push({ text: processedText, metadata }); } else { processedLine += text; + textAndMetadata.push({ text, metadata }); } }); - const processedTokens = new Uint32Array(processedTokensArray); - const processedLineTokens = new LineTokens(processedTokens, processedLine, tokens.languageIdCodec); + const processedLineTokens = LineTokens.createFromTextAndMetadata(textAndMetadata, tokens.languageIdCodec); return processedLineTokens; } } diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 12b77bcbab878..d1ee0f40365fb 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -49,6 +49,18 @@ export class LineTokens implements IViewLineTokens { return new LineTokens(tokens, lineContent, decoder); } + public static createFromTextAndMetadata(data: { text: string, metadata: number }[], decoder: ILanguageIdCodec): LineTokens { + let offset: number = 0; + let fullText: string = ''; + const tokens = new Array(); + for (const { text, metadata } of data) { + tokens.push(offset + text.length, metadata); + offset += text.length; + fullText += text; + } + return new LineTokens(new Uint32Array(tokens), fullText, decoder); + } + constructor(tokens: Uint32Array, text: string, decoder: ILanguageIdCodec) { this._tokens = tokens; this._tokensCount = (this._tokens.length >>> 1); From 623d169e80eb5a709b0cdd8c0b46c61e7129e7de Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 17:01:32 +0200 Subject: [PATCH 42/47] adding regexp directly into the bracket configuration --- .../supports/indentationLineProcessor.ts | 35 +++---------------- .../supports/languageBracketsConfiguration.ts | 6 ++++ .../languages/supports/richEditBrackets.ts | 2 +- 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index bd2ffbd37c4b7..acdfbcfcc52e3 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -210,44 +210,17 @@ class IndentationLineProcessor { */ getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { - // Utility functions + // Utility function const isTokenTypeFromWhichToRemoveBrackets = (tokenType: StandardTokenType): boolean => { return tokenType === StandardTokenType.String || tokenType === StandardTokenType.RegEx || tokenType === StandardTokenType.Comment; } - const removeBracketsFromText = (line: string): string => { - let processedLine = line; - openBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), 'g'); - processedLine = processedLine.replace(regex, ''); - }); - closedBrackets.forEach((bracket) => { - const regex = new RegExp(escapeStringForRegex(bracket), 'g'); - processedLine = processedLine.replace(regex, ''); - }); - return processedLine; - } - const escapeStringForRegex = (text: string): string => { - let res = ''; - for (const chr of text) { - res += escapeCharacterForRegex(chr); - } - return res; - }; - const escapeCharacterForRegex = (character: string): string => { - return character.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - } // Main code const languageId = tokens.getLanguageId(0); - const brackets = this.languageConfigurationService.getLanguageConfiguration(languageId).brackets; - if (!brackets) { - return tokens; - } - const openBrackets = brackets.brackets.map((brackets) => brackets.open).flat(); - const closedBrackets = brackets.brackets.map((brackets) => brackets.close).flat(); - + const bracketsConfiguration = this.languageConfigurationService.getLanguageConfiguration(languageId).bracketsNew; + const bracketsRegExp = bracketsConfiguration.getBracketRegExp(); let processedLine = ''; const textAndMetadata: { text: string, metadata: number }[] = []; tokens.forEach((tokenIndex: number) => { @@ -255,7 +228,7 @@ class IndentationLineProcessor { const text = tokens.getTokenText(tokenIndex); const metadata = tokens.getMetadata(tokenIndex); if (isTokenTypeFromWhichToRemoveBrackets(tokenType)) { - const processedText = removeBracketsFromText(text); + const processedText = text.replace(bracketsRegExp, ''); processedLine += processedText; textAndMetadata.push({ text: processedText, metadata }); } else { diff --git a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts index a989e2f35e41b..e4d2ee4ab0506 100644 --- a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts +++ b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts @@ -5,6 +5,7 @@ import { CachedFunction } from 'vs/base/common/cache'; import { LanguageConfiguration } from 'vs/editor/common/languages/languageConfiguration'; +import { createBracketOrRegExp } from 'vs/editor/common/languages/supports/richEditBrackets'; /** * Captures all bracket related configurations for a single language. @@ -91,6 +92,11 @@ export class LanguageBracketsConfiguration { public getBracketInfo(bracketText: string): BracketKind | undefined { return this.getOpeningBracketInfo(bracketText) || this.getClosingBracketInfo(bracketText); } + + public getBracketRegExp(): RegExp { + const brackets = Array.from([...this._openingBrackets.keys(), ...this._closingBrackets.keys()]); + return createBracketOrRegExp(brackets); + } } function filterValidBrackets(bracketPairs: [string, string][]): [string, string][] { diff --git a/src/vs/editor/common/languages/supports/richEditBrackets.ts b/src/vs/editor/common/languages/supports/richEditBrackets.ts index c6efd4ee7a780..dab2cf1304e1a 100644 --- a/src/vs/editor/common/languages/supports/richEditBrackets.ts +++ b/src/vs/editor/common/languages/supports/richEditBrackets.ts @@ -408,7 +408,7 @@ function prepareBracketForRegExp(str: string): string { return (insertWordBoundaries ? `\\b${str}\\b` : str); } -function createBracketOrRegExp(pieces: string[]): RegExp { +export function createBracketOrRegExp(pieces: string[]): RegExp { const regexStr = `(${pieces.map(prepareBracketForRegExp).join(')|(')})`; return strings.createRegExp(regexStr, true); } From f1845523a7b70759deb153f670789fd1f8e361e2 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 14 May 2024 17:45:07 +0200 Subject: [PATCH 43/47] polishing the code --- src/vs/editor/common/languages/autoIndent.ts | 12 ++++++------ src/vs/editor/common/languages/enterAction.ts | 6 +++--- .../languages/supports/indentationLineProcessor.ts | 7 ++++--- src/vs/editor/common/tokens/lineTokens.ts | 6 ++---- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 99888647363ff..cba8670214eb9 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -325,19 +325,19 @@ export function getIndentForEnter( const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedTokens.getLineContent()); const virtualModel = createVirtualModelWithModifiedTokensAtLine(model, range.startLineNumber, beforeEnterProcessedTokens); - const embeddedLanguage = isLanguageDifferentFromLineStart(model, range.getStartPosition()); + const languageIsDifferentFromLineStart = isLanguageDifferentFromLineStart(model, range.getStartPosition()); const currentLine = model.getLineContent(range.startLineNumber); const currentLineIndent = strings.getLeadingWhitespace(currentLine); const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); if (!afterEnterAction) { - const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; + const beforeEnter = languageIsDifferentFromLineStart ? currentLineIndent : beforeEnterIndent; return { beforeEnter: beforeEnter, afterEnter: beforeEnter }; } - let afterEnterIndent = embeddedLanguage ? currentLineIndent : afterEnterAction.indentation; + let afterEnterIndent = languageIsDifferentFromLineStart ? currentLineIndent : afterEnterAction.indentation; if (afterEnterAction.action === IndentAction.Indent) { afterEnterIndent = indentConverter.shiftIndent(afterEnterIndent); @@ -348,7 +348,7 @@ export function getIndentForEnter( } return { - beforeEnter: embeddedLanguage ? currentLineIndent : beforeEnterIndent, + beforeEnter: languageIsDifferentFromLineStart ? currentLineIndent : beforeEnterIndent, afterEnter: afterEnterIndent }; } @@ -456,6 +456,6 @@ function isLanguageDifferentFromLineStart(model: ITextModel, position: Position) const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; - const isWithinEmbeddedLanguage = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; - return isWithinEmbeddedLanguage; + const languageIsDifferentFromLineStart = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; + return languageIsDifferentFromLineStart; } diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index be290b7b090f3..27669db6ebe42 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -25,10 +25,10 @@ export function getEnterAction( const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); const previousLineText = processedContextTokens.previousLineProcessedTokens.getLineContent(); - const beforeRangeText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); - const afterRangeText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); + const beforeEnterText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); + const afterEnterText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); - const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeRangeText, afterRangeText); + const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText); if (!enterResult) { return null; } diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index acdfbcfcc52e3..bacc8f7ba5589 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -211,7 +211,7 @@ class IndentationLineProcessor { getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { // Utility function - const isTokenTypeFromWhichToRemoveBrackets = (tokenType: StandardTokenType): boolean => { + const shouldRemoveBracketsFromTokenType = (tokenType: StandardTokenType): boolean => { return tokenType === StandardTokenType.String || tokenType === StandardTokenType.RegEx || tokenType === StandardTokenType.Comment; @@ -221,13 +221,14 @@ class IndentationLineProcessor { const languageId = tokens.getLanguageId(0); const bracketsConfiguration = this.languageConfigurationService.getLanguageConfiguration(languageId).bracketsNew; const bracketsRegExp = bracketsConfiguration.getBracketRegExp(); - let processedLine = ''; const textAndMetadata: { text: string, metadata: number }[] = []; + + let processedLine = ''; tokens.forEach((tokenIndex: number) => { const tokenType = tokens.getStandardTokenType(tokenIndex); const text = tokens.getTokenText(tokenIndex); const metadata = tokens.getMetadata(tokenIndex); - if (isTokenTypeFromWhichToRemoveBrackets(tokenType)) { + if (shouldRemoveBracketsFromTokenType(tokenType)) { const processedText = text.replace(bracketsRegExp, ''); processedLine += processedText; textAndMetadata.push({ text: processedText, metadata }); diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index d1ee0f40365fb..767e610c9bec4 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -352,12 +352,10 @@ class SliceLineTokens implements IViewLineTokens { const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); let text = this._source.getTokenText(tokenIndex) if (tokenStartOffset < this._startOffset) { - const offsetDifference = this._startOffset - tokenStartOffset; - text = text.substring(offsetDifference); + text = text.substring(this._startOffset - tokenStartOffset); } if (tokenEndOffset > this._endOffset) { - const offsetDifference = tokenEndOffset - this._endOffset; - text = text.substring(0, text.length - offsetDifference); + text = text.substring(0, text.length - (tokenEndOffset - this._endOffset)); } return text; } From 903ddf22273b517b55baea6b429545c638332094 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Wed, 15 May 2024 08:05:41 +0200 Subject: [PATCH 44/47] adding the global flag on the regex --- .../common/languages/supports/indentationLineProcessor.ts | 2 +- .../languages/supports/languageBracketsConfiguration.ts | 5 +++-- src/vs/editor/common/languages/supports/richEditBrackets.ts | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index bacc8f7ba5589..23e03de745a44 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -220,7 +220,7 @@ class IndentationLineProcessor { // Main code const languageId = tokens.getLanguageId(0); const bracketsConfiguration = this.languageConfigurationService.getLanguageConfiguration(languageId).bracketsNew; - const bracketsRegExp = bracketsConfiguration.getBracketRegExp(); + const bracketsRegExp = bracketsConfiguration.getBracketRegExp({ global: true }); const textAndMetadata: { text: string, metadata: number }[] = []; let processedLine = ''; diff --git a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts index e4d2ee4ab0506..4989395b2646b 100644 --- a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts +++ b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { CachedFunction } from 'vs/base/common/cache'; +import { RegExpOptions } from 'vs/base/common/strings'; import { LanguageConfiguration } from 'vs/editor/common/languages/languageConfiguration'; import { createBracketOrRegExp } from 'vs/editor/common/languages/supports/richEditBrackets'; @@ -93,9 +94,9 @@ export class LanguageBracketsConfiguration { return this.getOpeningBracketInfo(bracketText) || this.getClosingBracketInfo(bracketText); } - public getBracketRegExp(): RegExp { + public getBracketRegExp(options?: RegExpOptions): RegExp { const brackets = Array.from([...this._openingBrackets.keys(), ...this._closingBrackets.keys()]); - return createBracketOrRegExp(brackets); + return createBracketOrRegExp(brackets, options); } } diff --git a/src/vs/editor/common/languages/supports/richEditBrackets.ts b/src/vs/editor/common/languages/supports/richEditBrackets.ts index dab2cf1304e1a..7733719f04975 100644 --- a/src/vs/editor/common/languages/supports/richEditBrackets.ts +++ b/src/vs/editor/common/languages/supports/richEditBrackets.ts @@ -7,6 +7,7 @@ import * as strings from 'vs/base/common/strings'; import * as stringBuilder from 'vs/editor/common/core/stringBuilder'; import { Range } from 'vs/editor/common/core/range'; import { CharacterPair } from 'vs/editor/common/languages/languageConfiguration'; +import { RegExpOptions } from 'vs/base/common/strings'; interface InternalBracket { open: string[]; @@ -408,9 +409,9 @@ function prepareBracketForRegExp(str: string): string { return (insertWordBoundaries ? `\\b${str}\\b` : str); } -export function createBracketOrRegExp(pieces: string[]): RegExp { +export function createBracketOrRegExp(pieces: string[], options?: RegExpOptions): RegExp { const regexStr = `(${pieces.map(prepareBracketForRegExp).join(')|(')})`; - return strings.createRegExp(regexStr, true); + return strings.createRegExp(regexStr, true, options); } const toReversedString = (function () { From 4d7495318f129cc576524461f06d1c1b05251290 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Wed, 15 May 2024 08:47:08 +0200 Subject: [PATCH 45/47] adding the text --- src/vs/editor/common/tokens/lineTokens.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 767e610c9bec4..21c87b70d4528 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -348,9 +348,10 @@ class SliceLineTokens implements IViewLineTokens { } public getTokenText(tokenIndex: number): string { - const tokenStartOffset = this._source.getStartOffset(this._firstTokenIndex + tokenIndex); - const tokenEndOffset = this._source.getEndOffset(this._firstTokenIndex + tokenIndex); - let text = this._source.getTokenText(tokenIndex) + const adjustedTokenIndex = this._firstTokenIndex + tokenIndex; + const tokenStartOffset = this._source.getStartOffset(adjustedTokenIndex); + const tokenEndOffset = this._source.getEndOffset(adjustedTokenIndex); + let text = this._source.getTokenText(adjustedTokenIndex); if (tokenStartOffset < this._startOffset) { text = text.substring(this._startOffset - tokenStartOffset); } @@ -361,9 +362,8 @@ class SliceLineTokens implements IViewLineTokens { } public forEach(callback: (tokenIndex: number) => void): void { - const tokenCount = this.getCount(); - const firstTokenIndex = this._firstTokenIndex; - const lastTokenIndex = this._firstTokenIndex + tokenCount; + const firstTokenIndex = 0; + const lastTokenIndex = this.getCount(); for (let tokenIndex = firstTokenIndex; tokenIndex < lastTokenIndex; tokenIndex++) { callback(tokenIndex); } From 01e76d6db9534fd4a5095a4c88eb770f07c0046b Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Wed, 22 May 2024 16:12:28 +0200 Subject: [PATCH 46/47] polishing the code and adding test for the indentation context processor --- src/vs/editor/common/languages/autoIndent.ts | 12 +- src/vs/editor/common/languages/supports.ts | 10 +- .../supports/indentationLineProcessor.ts | 111 ++++++++---------- src/vs/editor/common/tokens/lineTokens.ts | 6 +- .../test/browser/indentation.test.ts | 10 +- .../browser/indentationLineProcessor.test.ts | 57 +++++++++ 6 files changed, 125 insertions(+), 81 deletions(-) create mode 100644 src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index cba8670214eb9..4df9898986043 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -11,9 +11,7 @@ import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; -import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; -import { Position } from 'vs/editor/common/core/position'; +import { IndentationContextProcessor, isLanguageDifferentFromLineStart, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export interface IVirtualModel { tokenization: { @@ -451,11 +449,3 @@ function createVirtualModelWithModifiedTokensAtLine(model: ITextModel, modifiedL return virtualModel; } -function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { - const lineTokens = model.tokenization.getLineTokens(position.lineNumber); - const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); - const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; - const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; - const languageIsDifferentFromLineStart = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; - return languageIsDifferentFromLineStart; -} diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 39eeaff67e97e..730fa2a1b73b1 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; import { ILanguageIdCodec } from 'vs/editor/common/languages'; @@ -65,6 +65,10 @@ export class ScopedLineTokens { return actualLineContent.substring(this.firstCharOffset, this._lastCharOffset); } + public getLineLength(): number { + return this._lastCharOffset - this.firstCharOffset; + } + public getActualLineContentBefore(offset: number): string { const actualLineContent = this._actual.getLineContent(); return actualLineContent.substring(0, this.firstCharOffset + offset); @@ -81,6 +85,10 @@ export class ScopedLineTokens { public getStandardTokenType(tokenIndex: number): StandardTokenType { return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } + + public toIViewLineTokens(): IViewLineTokens { + return this._actual.sliceAndInflate(this.firstCharOffset, this._lastCharOffset, 0); + } } const enum IgnoreBracketsInTokens { diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index 23e03de745a44..fa73db599b4da 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -12,6 +12,7 @@ import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { Position } from 'vs/editor/common/core/position'; /** * This class is a wrapper class around {@link IndentRulesSupport}. @@ -94,63 +95,55 @@ export class IndentationContextProcessor { afterRangeProcessedTokens: IViewLineTokens; previousLineProcessedTokens: IViewLineTokens; } { - this.model.tokenization.forceTokenization(range.startLineNumber); - const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const beforeRangeProcessedTokens = this._getProcessedTokensBeforeRange(range, scopedLineTokens); - const afterRangeProcessedTokens = this._getProcessedTokensAfterRange(range, scopedLineTokens); - const previousLineProcessedTokens = this._getProcessedPreviousLineTokens(range, scopedLineTokens); + const beforeRangeProcessedTokens = this._getProcessedTokensBeforeRange(range); + const afterRangeProcessedTokens = this._getProcessedTokensAfterRange(range); + const previousLineProcessedTokens = this._getProcessedPreviousLineTokens(range); return { beforeRangeProcessedTokens, afterRangeProcessedTokens, previousLineProcessedTokens }; } - private _getProcessedTokensBeforeRange(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { + private _getProcessedTokensBeforeRange(range: Range): IViewLineTokens { + this.model.tokenization.forceTokenization(range.startLineNumber); const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; - const firstCharacterOffset = scopedLineTokens.firstCharOffset; - const lastCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const slicedTokensBefore = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokensBefore); + const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); + let slicedTokens: IViewLineTokens; + if (isLanguageDifferentFromLineStart(this.model, range.getStartPosition())) { + const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; + const firstCharacterOffset = scopedLineTokens.firstCharOffset; + const lastCharacterOffset = firstCharacterOffset + columnIndexWithinScope; + slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + } else { + const columnWithinLine = range.startColumn - 1; + slicedTokens = lineTokens.sliceAndInflate(0, columnWithinLine, 0); + } + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokens); return processedTokens; } - private _getProcessedTokensAfterRange(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { - let columnIndexWithinScope: number; - let lineTokens: LineTokens; - if (range.isEmpty()) { - columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; - lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); - } else { - columnIndexWithinScope = (range.endColumn - 1) - scopedLineTokens.firstCharOffset; - lineTokens = this.model.tokenization.getLineTokens(range.endLineNumber); - } - const scopedLineContent = scopedLineTokens.getLineContent(); + private _getProcessedTokensAfterRange(range: Range): IViewLineTokens { + const position: Position = range.isEmpty() ? range.getStartPosition() : range.getEndPosition(); + this.model.tokenization.forceTokenization(position.lineNumber); + const lineTokens = this.model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + const columnIndexWithinScope = position.column - 1 - scopedLineTokens.firstCharOffset; const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; - const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineContent.length; - const slicedTokensAfter = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokensAfter); + const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineTokens.getLineLength(); + const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokens); return processedTokens; } - private _getProcessedPreviousLineTokens(range: Range, scopedLineTokens: ScopedLineTokens): IViewLineTokens { - - // Utility functions + private _getProcessedPreviousLineTokens(range: Range): IViewLineTokens { const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { this.model.tokenization.forceTokenization(lineNumber); const lineTokens = this.model.tokenization.getLineTokens(lineNumber); const endColumnOfLine = this.model.getLineMaxColumn(lineNumber) - 1; const scopedLineTokensAtEndColumn = createScopedLineTokens(lineTokens, endColumnOfLine); return scopedLineTokensAtEndColumn; - } - const getSlicedLineTokensForScopeAtLine = (scopedLineTokens: ScopedLineTokens, lineNumber: number): IViewLineTokens => { - const initialLine = this.model.tokenization.getLineTokens(lineNumber); - const scopedLine = scopedLineTokens.getLineContent(); - const firstCharacterOffset = scopedLineTokens.firstCharOffset; - const lastCharacterOffset = firstCharacterOffset + scopedLine.length; - const slicedLineTokens = initialLine.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); - return slicedLineTokens; - } + }; - // Main code + this.model.tokenization.forceTokenization(range.startLineNumber); + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); const emptyTokens = LineTokens.createEmpty('', scopedLineTokens.languageIdCodec); const previousLineNumber = range.startLineNumber - 1; const isFirstLine = previousLineNumber === 0; @@ -166,7 +159,7 @@ export class IndentationContextProcessor { if (!doesLanguageContinueOnPreviousLine) { return emptyTokens; } - const previousSlicedLineTokens = getSlicedLineTokensForScopeAtLine(scopedLineTokensAtEndColumnOfPreviousLine, previousLineNumber); + const previousSlicedLineTokens = scopedLineTokensAtEndColumnOfPreviousLine.toIViewLineTokens(); const processedTokens = this.indentationLineProcessor.getProcessedTokens(previousSlicedLineTokens); return processedTokens; } @@ -188,19 +181,16 @@ class IndentationLineProcessor { * Remove the language configuration brackets from the regex, string and comment tokens. */ getProcessedLine(lineNumber: number, newIndentation?: string): string { - - // Utility function - const adjustIndentation = (line: string, newIndentation: string): string => { + const replaceIndentation = (line: string, newIndentation: string): string => { const currentIndentation = strings.getLeadingWhitespace(line); const adjustedLine = newIndentation + line.substring(currentIndentation.length); return adjustedLine; - } + }; - // Main code const tokens = this.model.tokenization.getLineTokens(lineNumber); let processedLine = this.getProcessedTokens(tokens).getLineContent(); if (newIndentation !== undefined) { - processedLine = adjustIndentation(processedLine, newIndentation); + processedLine = replaceIndentation(processedLine, newIndentation); } return processedLine; } @@ -210,34 +200,35 @@ class IndentationLineProcessor { */ getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { - // Utility function const shouldRemoveBracketsFromTokenType = (tokenType: StandardTokenType): boolean => { return tokenType === StandardTokenType.String || tokenType === StandardTokenType.RegEx || tokenType === StandardTokenType.Comment; - } + }; - // Main code const languageId = tokens.getLanguageId(0); const bracketsConfiguration = this.languageConfigurationService.getLanguageConfiguration(languageId).bracketsNew; const bracketsRegExp = bracketsConfiguration.getBracketRegExp({ global: true }); - const textAndMetadata: { text: string, metadata: number }[] = []; - - let processedLine = ''; + const textAndMetadata: { text: string; metadata: number }[] = []; tokens.forEach((tokenIndex: number) => { const tokenType = tokens.getStandardTokenType(tokenIndex); - const text = tokens.getTokenText(tokenIndex); - const metadata = tokens.getMetadata(tokenIndex); + let text = tokens.getTokenText(tokenIndex); if (shouldRemoveBracketsFromTokenType(tokenType)) { - const processedText = text.replace(bracketsRegExp, ''); - processedLine += processedText; - textAndMetadata.push({ text: processedText, metadata }); - } else { - processedLine += text; - textAndMetadata.push({ text, metadata }); + text = text.replace(bracketsRegExp, ''); } + const metadata = tokens.getMetadata(tokenIndex); + textAndMetadata.push({ text, metadata }); }); const processedLineTokens = LineTokens.createFromTextAndMetadata(textAndMetadata, tokens.languageIdCodec); return processedLineTokens; } } + +export function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { + const lineTokens = model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; + const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; + const languageIsDifferentFromLineStart = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; + return languageIsDifferentFromLineStart; +} diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index 21c87b70d4528..43777995d5100 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -49,7 +49,7 @@ export class LineTokens implements IViewLineTokens { return new LineTokens(tokens, lineContent, decoder); } - public static createFromTextAndMetadata(data: { text: string, metadata: number }[], decoder: ILanguageIdCodec): LineTokens { + public static createFromTextAndMetadata(data: { text: string; metadata: number }[], decoder: ILanguageIdCodec): LineTokens { let offset: number = 0; let fullText: string = ''; const tokens = new Array(); @@ -362,9 +362,7 @@ class SliceLineTokens implements IViewLineTokens { } public forEach(callback: (tokenIndex: number) => void): void { - const firstTokenIndex = 0; - const lastTokenIndex = this.getCount(); - for (let tokenIndex = firstTokenIndex; tokenIndex < lastTokenIndex; tokenIndex++) { + for (let tokenIndex = 0; tokenIndex < this.getCount(); tokenIndex++) { callback(tokenIndex); } } diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 1097b1bb316f4..0c126b7150394 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -24,7 +24,7 @@ import { TypeOperations } from 'vs/editor/common/cursor/cursorTypeOperations'; import { cppBracketRules, goBracketRules, htmlBracketRules, latexBracketRules, luaBracketRules, phpBracketRules, rubyBracketRules, typescriptBracketRules, vbBracketRules } from 'vs/editor/test/common/modes/supports/bracketRules'; import { latexAutoClosingPairsRules } from 'vs/editor/test/common/modes/supports/autoClosingPairsRules'; -enum Language { +export enum Language { TypeScript = 'ts-test', Ruby = 'ruby-test', PHP = 'php-test', @@ -44,7 +44,7 @@ function testIndentationToTabsCommand(lines: string[], selection: Selection, tab testCommand(lines, null, selection, (accessor, sel) => new IndentationToTabsCommand(sel, tabSize), expectedLines, expectedSelection); } -function registerLanguage(instantiationService: TestInstantiationService, language: Language): IDisposable { +export function registerLanguage(instantiationService: TestInstantiationService, language: Language): IDisposable { const disposables = new DisposableStore(); const languageService = instantiationService.get(ILanguageService); disposables.add(registerLanguageConfiguration(instantiationService, language)); @@ -52,7 +52,7 @@ function registerLanguage(instantiationService: TestInstantiationService, langua return disposables; } -function registerLanguageConfiguration(instantiationService: TestInstantiationService, language: Language): IDisposable { +export function registerLanguageConfiguration(instantiationService: TestInstantiationService, language: Language): IDisposable { const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); switch (language) { case Language.TypeScript: @@ -110,12 +110,12 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -interface StandardTokenTypeData { +export interface StandardTokenTypeData { startIndex: number; standardTokenType: StandardTokenType; } -function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: StandardTokenTypeData[][], languageId: string): IDisposable { +export function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: StandardTokenTypeData[][], languageId: string): IDisposable { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); const tokenizationSupport: ITokenizationSupport = { diff --git a/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts new file mode 100644 index 0000000000000..78d82542b2758 --- /dev/null +++ b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { Language, registerLanguage, registerTokenizationSupport, StandardTokenTypeData } from 'vs/editor/contrib/indentation/test/browser/indentation.test'; +import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor'; +import { createTextModel } from 'vs/editor/test/common/testTextModel'; +import { Range } from 'vs/editor/common/core/range'; + +suite('Indentation Context Processor - TypeScript/JavaScript', () => { + + const languageId = Language.TypeScript; + let disposables: DisposableStore; + + setup(() => { + disposables = new DisposableStore(); + }); + + teardown(() => { + disposables.dispose(); + }); + + ensureNoDisposablesAreLeakedInTestSuite(); + + test('issue #208232: incorrect indentation inside of comments', () => { + + // https://github.com/microsoft/vscode/issues/208232 + + const model = createTextModel([ + 'const someVar = "{some text}"', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [[ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 16, standardTokenType: StandardTokenType.String }, + { startIndex: 28, standardTokenType: StandardTokenType.String } + ]]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(1, 23, 1, 23)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someVar = "some'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), ' text"'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), ''); + }); + }); + +}); From b143fda00e779881d04a6e10c7a7a3bb5e02b091 Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Thu, 23 May 2024 11:53:41 +0200 Subject: [PATCH 47/47] adding tests --- src/vs/editor/common/languages/autoIndent.ts | 1 + .../supports/indentationLineProcessor.ts | 2 + .../test/browser/indentation.test.ts | 4 - .../browser/indentationLineProcessor.test.ts | 187 +++++++++++++++++- 4 files changed, 186 insertions(+), 8 deletions(-) diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 4df9898986043..680d3a7d5f9f0 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -18,6 +18,7 @@ export interface IVirtualModel { getLineTokens(lineNumber: number): IViewLineTokens; getLanguageId(): string; getLanguageIdAtPosition(lineNumber: number, column: number): string; + forceTokenization?(lineNumber: number): void; }; getLineContent(lineNumber: number): string; } diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts index fa73db599b4da..919cb3cd4c8ba 100644 --- a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -187,6 +187,7 @@ class IndentationLineProcessor { return adjustedLine; }; + this.model.tokenization.forceTokenization?.(lineNumber); const tokens = this.model.tokenization.getLineTokens(lineNumber); let processedLine = this.getProcessedTokens(tokens).getLineContent(); if (newIndentation !== undefined) { @@ -225,6 +226,7 @@ class IndentationLineProcessor { } export function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { + model.tokenization.forceTokenization(position.lineNumber); const lineTokens = model.tokenization.getLineTokens(position.lineNumber); const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 0c126b7150394..178e1db4061ba 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -1404,10 +1404,6 @@ suite('Auto Indent On Type - PHP', () => { ensureNoDisposablesAreLeakedInTestSuite(); - test('temp issue because there should be at least one passing test in a suite', () => { - assert.ok(true); - }); - test('issue #199050: should not indent after { detected in a string', () => { // https://github.com/microsoft/vscode/issues/199050 diff --git a/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts index 78d82542b2758..77afbe5c76d40 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts @@ -7,7 +7,7 @@ import { DisposableStore } from 'vs/base/common/lifecycle'; import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { Language, registerLanguage, registerTokenizationSupport, StandardTokenTypeData } from 'vs/editor/contrib/indentation/test/browser/indentation.test'; import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor'; import { createTextModel } from 'vs/editor/test/common/testTextModel'; @@ -28,9 +28,7 @@ suite('Indentation Context Processor - TypeScript/JavaScript', () => { ensureNoDisposablesAreLeakedInTestSuite(); - test('issue #208232: incorrect indentation inside of comments', () => { - - // https://github.com/microsoft/vscode/issues/208232 + test('brackets inside of string', () => { const model = createTextModel([ 'const someVar = "{some text}"', @@ -54,4 +52,185 @@ suite('Indentation Context Processor - TypeScript/JavaScript', () => { }); }); + test('brackets inside of comment', () => { + + const model = createTextModel([ + 'const someVar2 = /*(a])*/', + 'const someVar = /* [()] some other t{e}xt() */ "some text"', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.Comment }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 16, standardTokenType: StandardTokenType.Comment }, + { startIndex: 46, standardTokenType: StandardTokenType.Other }, + { startIndex: 47, standardTokenType: StandardTokenType.String } + ]]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(2, 29, 2, 35)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someVar = /* some'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), ' text */ "some text"'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), 'const someVar2 = /*a*/'); + }); + }); + + test('brackets inside of regex', () => { + + const model = createTextModel([ + 'const someRegex2 = /(()))]/;', + 'const someRegex = /()a{h}{s}[(a}87(9a9()))]/;', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 19, standardTokenType: StandardTokenType.RegEx }, + { startIndex: 27, standardTokenType: StandardTokenType.Other }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 18, standardTokenType: StandardTokenType.RegEx }, + { startIndex: 44, standardTokenType: StandardTokenType.Other }, + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(1, 25, 2, 33)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someRegex2 = /'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), '879a9/;'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), ''); + }); + }); +}); + +suite('Processed Indent Rules Support - TypeScript/JavaScript', () => { + + const languageId = Language.TypeScript; + let disposables: DisposableStore; + + setup(() => { + disposables = new DisposableStore(); + }); + + teardown(() => { + disposables.dispose(); + }); + + ensureNoDisposablesAreLeakedInTestSuite(); + + test('should increase', () => { + + const model = createTextModel([ + 'const someVar = {', + 'const someVar2 = "{"', + 'const someVar3 = /*{*/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.String }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.Comment }, + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(3), false); + }); + }); + + test('should decrease', () => { + + const model = createTextModel([ + '}', + '"])some text}"', + '])*/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [{ startIndex: 0, standardTokenType: StandardTokenType.Other }], + [{ startIndex: 0, standardTokenType: StandardTokenType.String }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(3), false); + }); + }); + + test('should increase next line', () => { + + const model = createTextModel([ + 'if()', + 'const someString = "if()"', + 'const someRegex = /if()/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 19, standardTokenType: StandardTokenType.String } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 18, standardTokenType: StandardTokenType.RegEx } + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(3), false); + }); + }); });