From 9157c7f869219dbaf9a5a5607f099c00fe694a29 Mon Sep 17 00:00:00 2001 From: Henning Dieterichs Date: Fri, 22 Apr 2022 15:52:39 +0200 Subject: [PATCH] Moves symbols around. --- src/grammar.ts | 144 +++------------- src/main.ts | 46 +---- src/metadata.ts | 159 +++++++++++++++++ src/onigLib.ts | 51 ++++++ src/{grammarReader.ts => parseRawGrammar.ts} | 2 +- src/{types.ts => rawGrammar.ts} | 49 ------ src/registry.ts | 4 +- src/rule.ts | 3 +- src/tests/grammar.test.ts | 2 +- src/tests/onigLibs.ts | 2 +- src/tests/resolver.ts | 5 +- src/tests/themedTokenizer.ts | 2 +- src/tests/themes.test.ts | 6 +- src/tests/tokenization.test.ts | 3 +- src/theme.ts | 172 +++++-------------- src/utils.ts | 65 ++++++- 16 files changed, 367 insertions(+), 348 deletions(-) create mode 100644 src/metadata.ts create mode 100644 src/onigLib.ts rename src/{grammarReader.ts => parseRawGrammar.ts} (95%) rename src/{types.ts => rawGrammar.ts} (58%) diff --git a/src/grammar.ts b/src/grammar.ts index 62b224c4..8bde6d87 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -3,12 +3,14 @@ *--------------------------------------------------------*/ import { clone, mergeObjects } from './utils'; -import { IRawGrammar, IRawRepository, IRawRule, IOnigLib, IOnigCaptureIndex, OnigString, OnigScanner, FindOption } from './types'; +import { IOnigLib, IOnigCaptureIndex, OnigString, OnigScanner, FindOption } from './onigLib'; import { IRuleRegistry, IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, CompiledRule } from './rule'; import { createMatchers, Matcher } from './matcher'; -import { MetadataConsts, IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main'; +import { IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main'; import { DebugFlags, UseOnigurumaFindOptions } from './debug'; import { FontStyle, ThemeTrieElementRule } from './theme'; +import { OptionalStandardTokenType, StackElementMetadata, toOptionalTokenType } from './metadata'; +import { IRawGrammar, IRawRule, IRawRepository } from './rawGrammar'; declare let performance: { now: () => number } | undefined; const performanceNow = (function () { @@ -20,18 +22,26 @@ const performanceNow = (function () { } })(); -// Must have the same values as `StandardTokenType`! -export const enum OptionalStandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 3, - // Indicates that no token type is set. - NotSet = 8 -} - -export function createGrammar(scopeName: string, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, balancedBracketSelectors: BalancedBracketSelectors | null, grammarRepository: IGrammarRepository & IThemeProvider, onigLib: IOnigLib): Grammar { - return new Grammar(scopeName, grammar, initialLanguage, embeddedLanguages, tokenTypes, balancedBracketSelectors, grammarRepository, onigLib);//TODO +export function createGrammar( + scopeName: string, + grammar: IRawGrammar, + initialLanguage: number, + embeddedLanguages: IEmbeddedLanguagesMap | null, + tokenTypes: ITokenTypeMap | null, + balancedBracketSelectors: BalancedBracketSelectors | null, + grammarRepository: IGrammarRepository & IThemeProvider, + onigLib: IOnigLib +): Grammar { + return new Grammar( + scopeName, + grammar, + initialLanguage, + embeddedLanguages, + tokenTypes, + balancedBracketSelectors, + grammarRepository, + onigLib + ); //TODO } export interface IThemeProvider { @@ -1196,99 +1206,6 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo } } - -export class StackElementMetadata { - - public static toBinaryStr(metadata: number): string { - let r = metadata.toString(2); - while (r.length < 32) { - r = '0' + r; - } - return r; - } - - public static printMetadata(metadata: number): void { - const languageId = StackElementMetadata.getLanguageId(metadata); - const tokenType = StackElementMetadata.getTokenType(metadata); - const fontStyle = StackElementMetadata.getFontStyle(metadata); - const foreground = StackElementMetadata.getForeground(metadata); - const background = StackElementMetadata.getBackground(metadata); - - console.log({ - languageId: languageId, - tokenType: tokenType, - fontStyle: fontStyle, - foreground: foreground, - background: background, - }); - } - - public static getLanguageId(metadata: number): number { - return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET; - } - - public static getTokenType(metadata: number): StandardTokenType { - return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; - } - - public static containsBalancedBrackets(metadata: number): boolean { - return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0; - } - - public static getFontStyle(metadata: number): number { - return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; - } - - public static getForeground(metadata: number): number { - return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET; - } - - public static getBackground(metadata: number): number { - return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET; - } - - /** - * Updates the fields in `metadata`. - * A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is. - */ - public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, containsBalancedBrackets: boolean | null, fontStyle: FontStyle, foreground: number, background: number): number { - let _languageId = StackElementMetadata.getLanguageId(metadata); - let _tokenType = StackElementMetadata.getTokenType(metadata); - let _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets(metadata) ? 1 : 0; - let _fontStyle = StackElementMetadata.getFontStyle(metadata); - let _foreground = StackElementMetadata.getForeground(metadata); - let _background = StackElementMetadata.getBackground(metadata); - - if (languageId !== 0) { - _languageId = languageId; - } - if (tokenType !== OptionalStandardTokenType.NotSet) { - _tokenType = fromOptionalTokenType(tokenType); - } - if (containsBalancedBrackets !== null) { - _containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0; - } - if (fontStyle !== FontStyle.NotSet) { - _fontStyle = fontStyle; - } - if (foreground !== 0) { - _foreground = foreground; - } - if (background !== 0) { - _background = background; - } - - return ( - (_languageId << MetadataConsts.LANGUAGEID_OFFSET) - | (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) - | (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET) - | (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET) - | (_foreground << MetadataConsts.FOREGROUND_OFFSET) - | (_background << MetadataConsts.BACKGROUND_OFFSET) - ) >>> 0; - } -} - export class ScopeListElement { public readonly parent: ScopeListElement | null; @@ -1848,16 +1765,3 @@ class LineTokens { } } -function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType { - return standardType as any as OptionalStandardTokenType; -} - -function fromOptionalTokenType( - standardType: - | OptionalStandardTokenType.Other - | OptionalStandardTokenType.Comment - | OptionalStandardTokenType.String - | OptionalStandardTokenType.RegEx -): StandardTokenType { - return standardType as any as StandardTokenType; -} diff --git a/src/main.ts b/src/main.ts index 39b1674c..8d739dad 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,13 +2,14 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ +import { BalancedBracketSelectors, ScopeDependencyProcessor, StackElement as StackElementImpl } from './grammar'; +import * as grammarReader from './parseRawGrammar'; +import { IOnigLib } from './onigLib'; +import { IRawGrammar } from './rawGrammar'; import { SyncRegistry } from './registry'; -import * as grammarReader from './grammarReader'; import { Theme } from './theme'; -import { StackElement as StackElementImpl, ScopeDependencyProcessor, BalancedBracketSelectors } from './grammar'; -import { IRawGrammar, IOnigLib } from './types'; -export * from './types'; +export * from './onigLib'; /** * A single theme setting. @@ -213,43 +214,6 @@ export interface ITokenizeLineResult { readonly stoppedEarly: boolean; } -/** - * Helpers to manage the "collapsed" metadata of an entire StackElement stack. - * The following assumptions have been made: - * - languageId < 256 => needs 8 bits - * - unique color count < 512 => needs 9 bits - * - * The binary format is: - * - ------------------------------------------- - * 3322 2222 2222 1111 1111 1100 0000 0000 - * 1098 7654 3210 9876 5432 1098 7654 3210 - * - ------------------------------------------- - * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL - * - ------------------------------------------- - * - L = LanguageId (8 bits) - * - T = StandardTokenType (2 bits) - * - B = Balanced bracket (1 bit) - * - F = FontStyle (4 bits) - * - f = foreground color (9 bits) - * - b = background color (9 bits) - */ -export const enum MetadataConsts { - LANGUAGEID_MASK = 0b00000000000000000000000011111111, - TOKEN_TYPE_MASK = 0b00000000000000000000001100000000, - BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000, - FONT_STYLE_MASK = 0b00000000000000000111100000000000, - FOREGROUND_MASK = 0b00000000111111111000000000000000, - BACKGROUND_MASK = 0b11111111000000000000000000000000, - - LANGUAGEID_OFFSET = 0, - TOKEN_TYPE_OFFSET = 8, - BALANCED_BRACKETS_OFFSET = 10, - FONT_STYLE_OFFSET = 11, - FOREGROUND_OFFSET = 15, - BACKGROUND_OFFSET = 24 -} - export interface ITokenizeLineResult2 { /** * The tokens in binary format. Each token occupies two array indices. For token i: diff --git a/src/metadata.ts b/src/metadata.ts new file mode 100644 index 00000000..a04a9b5a --- /dev/null +++ b/src/metadata.ts @@ -0,0 +1,159 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { StandardTokenType } from "./main"; +import { FontStyle } from "./theme"; + +/** + * Helpers to manage the "collapsed" metadata of an entire StackElement stack. + * The following assumptions have been made: + * - languageId < 256 => needs 8 bits + * - unique color count < 512 => needs 9 bits + * + * The binary format is: + * - ------------------------------------------- + * 3322 2222 2222 1111 1111 1100 0000 0000 + * 1098 7654 3210 9876 5432 1098 7654 3210 + * - ------------------------------------------- + * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx + * bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL + * - ------------------------------------------- + * - L = LanguageId (8 bits) + * - T = StandardTokenType (2 bits) + * - B = Balanced bracket (1 bit) + * - F = FontStyle (4 bits) + * - f = foreground color (9 bits) + * - b = background color (9 bits) + */ + export const enum MetadataConsts { + LANGUAGEID_MASK = 0b00000000000000000000000011111111, + TOKEN_TYPE_MASK = 0b00000000000000000000001100000000, + BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000, + FONT_STYLE_MASK = 0b00000000000000000111100000000000, + FOREGROUND_MASK = 0b00000000111111111000000000000000, + BACKGROUND_MASK = 0b11111111000000000000000000000000, + + LANGUAGEID_OFFSET = 0, + TOKEN_TYPE_OFFSET = 8, + BALANCED_BRACKETS_OFFSET = 10, + FONT_STYLE_OFFSET = 11, + FOREGROUND_OFFSET = 15, + BACKGROUND_OFFSET = 24 +} + +export class StackElementMetadata { + + public static toBinaryStr(metadata: number): string { + let r = metadata.toString(2); + while (r.length < 32) { + r = '0' + r; + } + return r; + } + + public static printMetadata(metadata: number): void { + const languageId = StackElementMetadata.getLanguageId(metadata); + const tokenType = StackElementMetadata.getTokenType(metadata); + const fontStyle = StackElementMetadata.getFontStyle(metadata); + const foreground = StackElementMetadata.getForeground(metadata); + const background = StackElementMetadata.getBackground(metadata); + + console.log({ + languageId: languageId, + tokenType: tokenType, + fontStyle: fontStyle, + foreground: foreground, + background: background, + }); + } + + public static getLanguageId(metadata: number): number { + return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET; + } + + public static getTokenType(metadata: number): StandardTokenType { + return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; + } + + public static containsBalancedBrackets(metadata: number): boolean { + return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0; + } + + public static getFontStyle(metadata: number): number { + return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; + } + + public static getForeground(metadata: number): number { + return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET; + } + + public static getBackground(metadata: number): number { + return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET; + } + + /** + * Updates the fields in `metadata`. + * A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is. + */ + public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, containsBalancedBrackets: boolean | null, fontStyle: FontStyle, foreground: number, background: number): number { + let _languageId = StackElementMetadata.getLanguageId(metadata); + let _tokenType = StackElementMetadata.getTokenType(metadata); + let _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets(metadata) ? 1 : 0; + let _fontStyle = StackElementMetadata.getFontStyle(metadata); + let _foreground = StackElementMetadata.getForeground(metadata); + let _background = StackElementMetadata.getBackground(metadata); + + if (languageId !== 0) { + _languageId = languageId; + } + if (tokenType !== OptionalStandardTokenType.NotSet) { + _tokenType = fromOptionalTokenType(tokenType); + } + if (containsBalancedBrackets !== null) { + _containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0; + } + if (fontStyle !== FontStyle.NotSet) { + _fontStyle = fontStyle; + } + if (foreground !== 0) { + _foreground = foreground; + } + if (background !== 0) { + _background = background; + } + + return ( + (_languageId << MetadataConsts.LANGUAGEID_OFFSET) + | (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) + | (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET) + | (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET) + | (_foreground << MetadataConsts.FOREGROUND_OFFSET) + | (_background << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; + } +} + +export function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType { + return standardType as any as OptionalStandardTokenType; +} + +function fromOptionalTokenType( + standardType: + | OptionalStandardTokenType.Other + | OptionalStandardTokenType.Comment + | OptionalStandardTokenType.String + | OptionalStandardTokenType.RegEx +): StandardTokenType { + return standardType as any as StandardTokenType; +} + +// Must have the same values as `StandardTokenType`! +export const enum OptionalStandardTokenType { + Other = 0, + Comment = 1, + String = 2, + RegEx = 3, + // Indicates that no token type is set. + NotSet = 8 +} diff --git a/src/onigLib.ts b/src/onigLib.ts new file mode 100644 index 00000000..4932e8d6 --- /dev/null +++ b/src/onigLib.ts @@ -0,0 +1,51 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +import { OrMask } from "./utils"; + +export interface IOnigLib { + createOnigScanner(sources: string[]): OnigScanner; + createOnigString(str: string): OnigString; +} + +export interface IOnigCaptureIndex { + start: number; + end: number; + length: number; +} + +export interface IOnigMatch { + index: number; + captureIndices: IOnigCaptureIndex[]; +} + +export const enum FindOption { + None = 0, + /** + * equivalent of ONIG_OPTION_NOT_BEGIN_STRING: (str) isn't considered as begin of string (* fail \A) + */ + NotBeginString = 1, + /** + * equivalent of ONIG_OPTION_NOT_END_STRING: (end) isn't considered as end of string (* fail \z, \Z) + */ + NotEndString = 2, + /** + * equivalent of ONIG_OPTION_NOT_BEGIN_POSITION: (start) isn't considered as start position of search (* fail \G) + */ + NotBeginPosition = 4, + /** + * used for debugging purposes. + */ + DebugCall = 8, +} + +export interface OnigScanner { + findNextMatchSync(string: string | OnigString, startPosition: number, options: OrMask): IOnigMatch | null; + dispose?(): void; +} + +export interface OnigString { + readonly content: string; + dispose?(): void; +} diff --git a/src/grammarReader.ts b/src/parseRawGrammar.ts similarity index 95% rename from src/grammarReader.ts rename to src/parseRawGrammar.ts index c8ee05d1..a8bc43c6 100644 --- a/src/grammarReader.ts +++ b/src/parseRawGrammar.ts @@ -2,7 +2,7 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IRawGrammar } from './types'; +import { IRawGrammar } from './rawGrammar'; import * as plist from './plist'; import { DebugFlags } from './debug'; import { parse as manualParseJSON } from './json'; diff --git a/src/types.ts b/src/rawGrammar.ts similarity index 58% rename from src/types.ts rename to src/rawGrammar.ts index abcc2804..e0af7dce 100644 --- a/src/types.ts +++ b/src/rawGrammar.ts @@ -2,8 +2,6 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -// -- raw grammar typings - export interface ILocation { readonly filename: string; readonly line: number; @@ -62,50 +60,3 @@ export interface IRawCapturesMap { } export type IRawCaptures = IRawCapturesMap & ILocatable; - - -export interface IOnigLib { - createOnigScanner(sources: string[]): OnigScanner; - createOnigString(str: string): OnigString; -} - -export interface IOnigCaptureIndex { - start: number; - end: number; - length: number; -} - -export interface IOnigMatch { - index: number; - captureIndices: IOnigCaptureIndex[]; -} - -export const enum FindOption { - None = 0, - /** - * equivalent of ONIG_OPTION_NOT_BEGIN_STRING: (str) isn't considered as begin of string (* fail \A) - */ - NotBeginString = 1, - /** - * equivalent of ONIG_OPTION_NOT_END_STRING: (end) isn't considered as end of string (* fail \z, \Z) - */ - NotEndString = 2, - /** - * equivalent of ONIG_OPTION_NOT_BEGIN_POSITION: (start) isn't considered as start position of search (* fail \G) - */ - NotBeginPosition = 4, - /** - * used for debugging purposes. - */ - DebugCall = 8, -} - -export interface OnigScanner { - findNextMatchSync(string: string | OnigString, startPosition: number, options: number): IOnigMatch | null; - dispose?(): void; -} - -export interface OnigString { - readonly content: string; - dispose?(): void; -} diff --git a/src/registry.ts b/src/registry.ts index d2d23ef8..1ef75637 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -3,10 +3,10 @@ *--------------------------------------------------------*/ import { BalancedBracketSelectors, createGrammar, Grammar, IGrammarRepository } from './grammar'; -import { IRawGrammar } from './types'; +import { IRawGrammar } from './rawGrammar'; import { IGrammar, IEmbeddedLanguagesMap, ITokenTypeMap } from './main'; import { Theme, ThemeTrieElementRule } from './theme'; -import { IOnigLib } from './types'; +import { IOnigLib } from './onigLib'; export class SyncRegistry implements IGrammarRepository { diff --git a/src/rule.ts b/src/rule.ts index 15250bba..faf36296 100644 --- a/src/rule.ts +++ b/src/rule.ts @@ -3,7 +3,8 @@ *--------------------------------------------------------*/ import { RegexSource, mergeObjects, basename } from './utils'; -import { ILocation, IRawGrammar, IRawRepository, IRawRule, IRawCaptures, IOnigLib, OnigScanner, IOnigCaptureIndex } from './types'; +import { IOnigLib, OnigScanner, IOnigCaptureIndex } from './onigLib'; +import { ILocation, IRawGrammar, IRawRepository, IRawRule, IRawCaptures } from './rawGrammar'; const HAS_BACK_REFERENCES = /\\(\d+)/; const BACK_REFERENCING_END = /\\(\d+)/g; diff --git a/src/tests/grammar.test.ts b/src/tests/grammar.test.ts index 66f09575..a7d53599 100644 --- a/src/tests/grammar.test.ts +++ b/src/tests/grammar.test.ts @@ -4,7 +4,7 @@ import * as assert from 'assert'; import { StandardTokenType } from '../main'; -import { StackElementMetadata, OptionalStandardTokenType } from '../grammar'; +import { StackElementMetadata, OptionalStandardTokenType } from '../metadata'; import { FontStyle } from '../theme'; function assertEquals(metadata: number, languageId: number, tokenType: StandardTokenType, containsBalancedBrackets: boolean, fontStyle: FontStyle, foreground: number, background: number): void { diff --git a/src/tests/onigLibs.ts b/src/tests/onigLibs.ts index 29896f3c..a669a66f 100644 --- a/src/tests/onigLibs.ts +++ b/src/tests/onigLibs.ts @@ -2,7 +2,7 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IOnigLib } from '../types'; +import { IOnigLib } from '../onigLib'; let onigurumaLib: Promise | null = null; diff --git a/src/tests/resolver.ts b/src/tests/resolver.ts index 09d8d099..90a0072c 100644 --- a/src/tests/resolver.ts +++ b/src/tests/resolver.ts @@ -2,12 +2,13 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IRawGrammar, IOnigLib } from '../types'; -import { parseRawGrammar } from '../grammarReader'; +import { IOnigLib } from '../onigLib'; +import { parseRawGrammar } from '../parseRawGrammar'; import { RegistryOptions } from '../main'; import * as path from 'path'; import * as fs from 'fs'; +import { IRawGrammar } from '../rawGrammar'; export interface ILanguageRegistration { id: string; diff --git a/src/tests/themedTokenizer.ts b/src/tests/themedTokenizer.ts index 0b313ee6..bc67d022 100644 --- a/src/tests/themedTokenizer.ts +++ b/src/tests/themedTokenizer.ts @@ -3,7 +3,7 @@ *--------------------------------------------------------*/ import { IGrammar, StackElement } from '../main'; -import { StackElementMetadata } from '../grammar'; +import { StackElementMetadata } from '../metadata'; export interface IThemedToken { content: string; diff --git a/src/tests/themes.test.ts b/src/tests/themes.test.ts index 80b5a44d..a27f51a7 100644 --- a/src/tests/themes.test.ts +++ b/src/tests/themes.test.ts @@ -6,15 +6,17 @@ import * as fs from 'fs'; import * as path from 'path'; import * as assert from 'assert'; import { Registry, IRawTheme } from '../main'; -import { ScopeListElement, ScopeMetadata, StackElementMetadata } from '../grammar'; +import { ScopeListElement, ScopeMetadata } from '../grammar'; import { - Theme, strcmp, strArrCmp, ThemeTrieElement, ThemeTrieElementRule, + Theme, ThemeTrieElement, ThemeTrieElementRule, parseTheme, ParsedThemeRule, FontStyle, ColorMap } from '../theme'; import * as plist from '../plist'; import { ThemeTest } from './themeTest'; import { getOniguruma } from './onigLibs'; import { Resolver, IGrammarRegistration, ILanguageRegistration } from './resolver'; +import { StackElementMetadata } from '../metadata'; +import { strArrCmp, strcmp } from '../utils'; const THEMES_TEST_PATH = path.join(__dirname, '../../test-cases/themes'); diff --git a/src/tests/tokenization.test.ts b/src/tests/tokenization.test.ts index d54a69d4..320c5cc8 100644 --- a/src/tests/tokenization.test.ts +++ b/src/tests/tokenization.test.ts @@ -6,8 +6,9 @@ import * as fs from 'fs'; import * as path from 'path'; import * as assert from 'assert'; import { Registry, IGrammar, RegistryOptions, StackElement, parseRawGrammar } from '../main'; -import { IOnigLib, IRawGrammar } from '../types'; +import { IOnigLib } from '../onigLib'; import { getOniguruma } from './onigLibs'; +import { IRawGrammar } from '../rawGrammar'; const REPO_ROOT = path.join(__dirname, '../../'); diff --git a/src/theme.ts b/src/theme.ts index 666c4823..aa723791 100644 --- a/src/theme.ts +++ b/src/theme.ts @@ -3,68 +3,43 @@ *--------------------------------------------------------*/ import { IRawTheme } from './main'; +import { isValidHexColor, OrMask, strArrCmp, strcmp } from './utils'; -export const enum FontStyle { - NotSet = -1, - None = 0, - Italic = 1, - Bold = 2, - Underline = 4, - Strikethrough = 8 -} - -export class ParsedThemeRule { - - readonly scope: string; - readonly parentScopes: string[] | null; - readonly index: number; - - /** - * -1 if not set. An or mask of `FontStyle` otherwise. - */ - readonly fontStyle: number; - readonly foreground: string | null; - readonly background: string | null; - - constructor( - scope: string, - parentScopes: string[] | null, - index: number, - fontStyle: number, - foreground: string | null, - background: string | null, - ) { - this.scope = scope; - this.parentScopes = parentScopes; - this.index = index; - this.fontStyle = fontStyle; - this.foreground = foreground; - this.background = background; +export class Theme { + public static createFromRawTheme(source: IRawTheme | undefined, colorMap?: string[]): Theme { + return this.createFromParsedTheme(parseTheme(source), colorMap); } -} -function isValidHexColor(hex: string): boolean { - if (/^#[0-9a-f]{6}$/i.test(hex)) { - // #rrggbb - return true; + public static createFromParsedTheme(source: ParsedThemeRule[], colorMap?: string[]): Theme { + return resolveParsedThemeRules(source, colorMap); } - if (/^#[0-9a-f]{8}$/i.test(hex)) { - // #rrggbbaa - return true; + private readonly _colorMap: ColorMap; + private readonly _root: ThemeTrieElement; + private readonly _defaults: ThemeTrieElementRule; + private readonly _cache: { [scopeName: string]: ThemeTrieElementRule[]; }; + + constructor(colorMap: ColorMap, defaults: ThemeTrieElementRule, root: ThemeTrieElement) { + this._colorMap = colorMap; + this._root = root; + this._defaults = defaults; + this._cache = {}; } - if (/^#[0-9a-f]{3}$/i.test(hex)) { - // #rgb - return true; + public getColorMap(): string[] { + return this._colorMap.getColorMap(); } - if (/^#[0-9a-f]{4}$/i.test(hex)) { - // #rgba - return true; + public getDefaults(): ThemeTrieElementRule { + return this._defaults; } - return false; + public match(scopeName: string): ThemeTrieElementRule[] { + if (!this._cache.hasOwnProperty(scopeName)) { + this._cache[scopeName] = this._root.match(scopeName); + } + return this._cache[scopeName]; + } } /** @@ -163,6 +138,27 @@ export function parseTheme(source: IRawTheme | undefined): ParsedThemeRule[] { return result; } +export class ParsedThemeRule { + constructor( + public readonly scope: string, + public readonly parentScopes: string[] | null, + public readonly index: number, + public readonly fontStyle: OrMask, + public readonly foreground: string | null, + public readonly background: string | null, + ) { + } +} + +export const enum FontStyle { + NotSet = -1, + None = 0, + Italic = 1, + Bold = 2, + Underline = 4, + Strikethrough = 8 +} + /** * Resolve rules (i.e. inheritance). */ @@ -210,7 +206,6 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[], _colorMap: } export class ColorMap { - private readonly _isFrozen: boolean; private _lastColorId: number; private _id2color: string[]; @@ -253,79 +248,6 @@ export class ColorMap { public getColorMap(): string[] { return this._id2color.slice(0); } - -} - -export class Theme { - - public static createFromRawTheme(source: IRawTheme | undefined, colorMap?: string[]): Theme { - return this.createFromParsedTheme(parseTheme(source), colorMap); - } - - public static createFromParsedTheme(source: ParsedThemeRule[], colorMap?: string[]): Theme { - return resolveParsedThemeRules(source, colorMap); - } - - private readonly _colorMap: ColorMap; - private readonly _root: ThemeTrieElement; - private readonly _defaults: ThemeTrieElementRule; - private readonly _cache: { [scopeName: string]: ThemeTrieElementRule[]; }; - - constructor(colorMap: ColorMap, defaults: ThemeTrieElementRule, root: ThemeTrieElement) { - this._colorMap = colorMap; - this._root = root; - this._defaults = defaults; - this._cache = {}; - } - - public getColorMap(): string[] { - return this._colorMap.getColorMap(); - } - - public getDefaults(): ThemeTrieElementRule { - return this._defaults; - } - - public match(scopeName: string): ThemeTrieElementRule[] { - if (!this._cache.hasOwnProperty(scopeName)) { - this._cache[scopeName] = this._root.match(scopeName); - } - return this._cache[scopeName]; - } -} - -export function strcmp(a: string, b: string): number { - if (a < b) { - return -1; - } - if (a > b) { - return 1; - } - return 0; -} - -export function strArrCmp(a: string[] | null, b: string[] | null): number { - if (a === null && b === null) { - return 0; - } - if (!a) { - return -1; - } - if (!b) { - return 1; - } - let len1 = a.length; - let len2 = b.length; - if (len1 === len2) { - for (let i = 0; i < len1; i++) { - let res = strcmp(a[i], b[i]); - if (res !== 0) { - return res; - } - } - return 0; - } - return len1 - len2; } export class ThemeTrieElementRule { diff --git a/src/utils.ts b/src/utils.ts index 7e44feaa..0dc5ee3f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,7 +2,7 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ -import { IOnigCaptureIndex } from './types'; +import { IOnigCaptureIndex } from './onigLib'; export function clone(something: T): T { return doClone(something); @@ -89,3 +89,66 @@ export class RegexSource { }); } } + +/** + * A union of given const enum values. +*/ +export type OrMask = number; + +export function strcmp(a: string, b: string): number { + if (a < b) { + return -1; + } + if (a > b) { + return 1; + } + return 0; +} + +export function strArrCmp(a: string[] | null, b: string[] | null): number { + if (a === null && b === null) { + return 0; + } + if (!a) { + return -1; + } + if (!b) { + return 1; + } + let len1 = a.length; + let len2 = b.length; + if (len1 === len2) { + for (let i = 0; i < len1; i++) { + let res = strcmp(a[i], b[i]); + if (res !== 0) { + return res; + } + } + return 0; + } + return len1 - len2; +} + +export function isValidHexColor(hex: string): boolean { + if (/^#[0-9a-f]{6}$/i.test(hex)) { + // #rrggbb + return true; + } + + if (/^#[0-9a-f]{8}$/i.test(hex)) { + // #rrggbbaa + return true; + } + + if (/^#[0-9a-f]{3}$/i.test(hex)) { + // #rgb + return true; + } + + if (/^#[0-9a-f]{4}$/i.test(hex)) { + // #rgba + return true; + } + + return false; +}