respect editor.maxTokenizationLineLength preference

packages/monaco/src/browser/textmate/monaco-textmate-service.ts

@@ -23,13 +23,18 @@ import { LanguageGrammarDefinitionContribution, getEncodedLanguageId } from './t
 import { createTextmateTokenizer, TokenizerOption } from './textmate-tokenizer';
 import { TextmateRegistry } from './textmate-registry';
 import { MonacoThemeRegistry } from './monaco-theme-registry';
+import { EditorPreferences } from '@theia/editor/lib/browser/editor-preferences';
 
 export const OnigasmPromise = Symbol('OnigasmPromise');
 export type OnigasmPromise = Promise<IOnigLib>;
 
 @injectable()
 export class MonacoTextmateService implements FrontendApplicationContribution {
 
+    protected readonly tokenizerOption: TokenizerOption = {
+        lineLimit: 400
+    };
+
     protected readonly _activatedLanguages = new Set<string>();
 
     protected grammarRegistry: Registry;
@@ -52,6 +57,9 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
     @inject(MonacoThemeRegistry)
     protected readonly monacoThemeRegistry: MonacoThemeRegistry;
 
+    @inject(EditorPreferences)
+    protected readonly preferences: EditorPreferences;
+
     initialize(): void {
         if (!isBasicWasmSupported) {
             console.log('Textmate support deactivated because WebAssembly is not detected.');
@@ -92,6 +100,13 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
             }
         });
 
+        this.tokenizerOption.lineLimit = this.preferences['editor.maxTokenizationLineLength'];
+        this.preferences.onPreferenceChanged(e => {
+            if (e.preferenceName === 'editor.maxTokenizationLineLength') {
+                this.tokenizerOption.lineLimit = this.preferences['editor.maxTokenizationLineLength'];
+            }
+        });
+
         this.updateTheme();
         this.themeService.onThemeChange(() => this.updateTheme());
 
@@ -164,7 +179,7 @@ export class MonacoTextmateService implements FrontendApplicationContribution {
             if (!grammar) {
                 throw new Error(`no grammar for ${scopeName}, ${initialLanguage}, ${JSON.stringify(configuration)}`);
             }
-            const options = configuration.tokenizerOption ? configuration.tokenizerOption : TokenizerOption.DEFAULT;
+            const options = configuration.tokenizerOption ? configuration.tokenizerOption : this.tokenizerOption;
             const tokenizer = createTextmateTokenizer(grammar, options);
             toDispose.push(monaco.languages.setTokensProvider(languageId, tokenizer));
             const support = monaco.modes.TokenizationRegistry.get(languageId);

packages/monaco/src/browser/textmate/textmate-tokenizer.ts

@@ -44,13 +44,15 @@ export interface TokenizerOption {
      * If the `lineLimit` is not defined, it means, there are no line length limits. Otherwise, it must be a positive
      * integer or an error will be thrown.
      */
-    readonly lineLimit?: number;
+    lineLimit?: number;
 
 }
 
 export namespace TokenizerOption {
     /**
      * The default TextMate tokenizer option.
+     *
+     * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
      */
     export const DEFAULT: TokenizerOption = {
         lineLimit: 400