Skip to content

Commit

Permalink
Moves symbols around.
Browse files Browse the repository at this point in the history
  • Loading branch information
hediet committed Apr 22, 2022
1 parent ed93673 commit 9157c7f
Show file tree
Hide file tree
Showing 16 changed files with 367 additions and 348 deletions.
144 changes: 24 additions & 120 deletions src/grammar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@
*--------------------------------------------------------*/

import { clone, mergeObjects } from './utils';
import { IRawGrammar, IRawRepository, IRawRule, IOnigLib, IOnigCaptureIndex, OnigString, OnigScanner, FindOption } from './types';
import { IOnigLib, IOnigCaptureIndex, OnigString, OnigScanner, FindOption } from './onigLib';
import { IRuleRegistry, IRuleFactoryHelper, RuleFactory, Rule, CaptureRule, BeginEndRule, BeginWhileRule, MatchRule, CompiledRule } from './rule';
import { createMatchers, Matcher } from './matcher';
import { MetadataConsts, IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main';
import { IGrammar, ITokenizeLineResult, ITokenizeLineResult2, IToken, IEmbeddedLanguagesMap, StandardTokenType, StackElement as StackElementDef, ITokenTypeMap } from './main';
import { DebugFlags, UseOnigurumaFindOptions } from './debug';
import { FontStyle, ThemeTrieElementRule } from './theme';
import { OptionalStandardTokenType, StackElementMetadata, toOptionalTokenType } from './metadata';
import { IRawGrammar, IRawRule, IRawRepository } from './rawGrammar';

declare let performance: { now: () => number } | undefined;
const performanceNow = (function () {
Expand All @@ -20,18 +22,26 @@ const performanceNow = (function () {
}
})();

// Must have the same values as `StandardTokenType`!
export const enum OptionalStandardTokenType {
Other = 0,
Comment = 1,
String = 2,
RegEx = 3,
// Indicates that no token type is set.
NotSet = 8
}

export function createGrammar(scopeName: string, grammar: IRawGrammar, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap | null, tokenTypes: ITokenTypeMap | null, balancedBracketSelectors: BalancedBracketSelectors | null, grammarRepository: IGrammarRepository & IThemeProvider, onigLib: IOnigLib): Grammar {
return new Grammar(scopeName, grammar, initialLanguage, embeddedLanguages, tokenTypes, balancedBracketSelectors, grammarRepository, onigLib);//TODO
export function createGrammar(
scopeName: string,
grammar: IRawGrammar,
initialLanguage: number,
embeddedLanguages: IEmbeddedLanguagesMap | null,
tokenTypes: ITokenTypeMap | null,
balancedBracketSelectors: BalancedBracketSelectors | null,
grammarRepository: IGrammarRepository & IThemeProvider,
onigLib: IOnigLib
): Grammar {
return new Grammar(
scopeName,
grammar,
initialLanguage,
embeddedLanguages,
tokenTypes,
balancedBracketSelectors,
grammarRepository,
onigLib
); //TODO
}

export interface IThemeProvider {
Expand Down Expand Up @@ -1196,99 +1206,6 @@ function _tokenizeString(grammar: Grammar, lineText: OnigString, isFirstLine: bo
}
}


export class StackElementMetadata {

public static toBinaryStr(metadata: number): string {
let r = metadata.toString(2);
while (r.length < 32) {
r = '0' + r;
}
return r;
}

public static printMetadata(metadata: number): void {
const languageId = StackElementMetadata.getLanguageId(metadata);
const tokenType = StackElementMetadata.getTokenType(metadata);
const fontStyle = StackElementMetadata.getFontStyle(metadata);
const foreground = StackElementMetadata.getForeground(metadata);
const background = StackElementMetadata.getBackground(metadata);

console.log({
languageId: languageId,
tokenType: tokenType,
fontStyle: fontStyle,
foreground: foreground,
background: background,
});
}

public static getLanguageId(metadata: number): number {
return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET;
}

public static getTokenType(metadata: number): StandardTokenType {
return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
}

public static containsBalancedBrackets(metadata: number): boolean {
return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0;
}

public static getFontStyle(metadata: number): number {
return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET;
}

public static getForeground(metadata: number): number {
return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET;
}

public static getBackground(metadata: number): number {
return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET;
}

/**
* Updates the fields in `metadata`.
* A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is.
*/
public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, containsBalancedBrackets: boolean | null, fontStyle: FontStyle, foreground: number, background: number): number {
let _languageId = StackElementMetadata.getLanguageId(metadata);
let _tokenType = StackElementMetadata.getTokenType(metadata);
let _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets(metadata) ? 1 : 0;
let _fontStyle = StackElementMetadata.getFontStyle(metadata);
let _foreground = StackElementMetadata.getForeground(metadata);
let _background = StackElementMetadata.getBackground(metadata);

if (languageId !== 0) {
_languageId = languageId;
}
if (tokenType !== OptionalStandardTokenType.NotSet) {
_tokenType = fromOptionalTokenType(tokenType);
}
if (containsBalancedBrackets !== null) {
_containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0;
}
if (fontStyle !== FontStyle.NotSet) {
_fontStyle = fontStyle;
}
if (foreground !== 0) {
_foreground = foreground;
}
if (background !== 0) {
_background = background;
}

return (
(_languageId << MetadataConsts.LANGUAGEID_OFFSET)
| (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET)
| (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET)
| (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
| (_foreground << MetadataConsts.FOREGROUND_OFFSET)
| (_background << MetadataConsts.BACKGROUND_OFFSET)
) >>> 0;
}
}

export class ScopeListElement {

public readonly parent: ScopeListElement | null;
Expand Down Expand Up @@ -1848,16 +1765,3 @@ class LineTokens {
}
}

function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType {
return standardType as any as OptionalStandardTokenType;
}

function fromOptionalTokenType(
standardType:
| OptionalStandardTokenType.Other
| OptionalStandardTokenType.Comment
| OptionalStandardTokenType.String
| OptionalStandardTokenType.RegEx
): StandardTokenType {
return standardType as any as StandardTokenType;
}
46 changes: 5 additions & 41 deletions src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@
* Copyright (C) Microsoft Corporation. All rights reserved.
*--------------------------------------------------------*/

import { BalancedBracketSelectors, ScopeDependencyProcessor, StackElement as StackElementImpl } from './grammar';
import * as grammarReader from './parseRawGrammar';
import { IOnigLib } from './onigLib';
import { IRawGrammar } from './rawGrammar';
import { SyncRegistry } from './registry';
import * as grammarReader from './grammarReader';
import { Theme } from './theme';
import { StackElement as StackElementImpl, ScopeDependencyProcessor, BalancedBracketSelectors } from './grammar';
import { IRawGrammar, IOnigLib } from './types';

export * from './types';
export * from './onigLib';

/**
* A single theme setting.
Expand Down Expand Up @@ -213,43 +214,6 @@ export interface ITokenizeLineResult {
readonly stoppedEarly: boolean;
}

/**
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
* The following assumptions have been made:
* - languageId < 256 => needs 8 bits
* - unique color count < 512 => needs 9 bits
*
* The binary format is:
* - -------------------------------------------
* 3322 2222 2222 1111 1111 1100 0000 0000
* 1098 7654 3210 9876 5432 1098 7654 3210
* - -------------------------------------------
* xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
* bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL
* - -------------------------------------------
* - L = LanguageId (8 bits)
* - T = StandardTokenType (2 bits)
* - B = Balanced bracket (1 bit)
* - F = FontStyle (4 bits)
* - f = foreground color (9 bits)
* - b = background color (9 bits)
*/
export const enum MetadataConsts {
LANGUAGEID_MASK = 0b00000000000000000000000011111111,
TOKEN_TYPE_MASK = 0b00000000000000000000001100000000,
BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000,
FONT_STYLE_MASK = 0b00000000000000000111100000000000,
FOREGROUND_MASK = 0b00000000111111111000000000000000,
BACKGROUND_MASK = 0b11111111000000000000000000000000,

LANGUAGEID_OFFSET = 0,
TOKEN_TYPE_OFFSET = 8,
BALANCED_BRACKETS_OFFSET = 10,
FONT_STYLE_OFFSET = 11,
FOREGROUND_OFFSET = 15,
BACKGROUND_OFFSET = 24
}

export interface ITokenizeLineResult2 {
/**
* The tokens in binary format. Each token occupies two array indices. For token i:
Expand Down
159 changes: 159 additions & 0 deletions src/metadata.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
/*---------------------------------------------------------
* Copyright (C) Microsoft Corporation. All rights reserved.
*--------------------------------------------------------*/

import { StandardTokenType } from "./main";
import { FontStyle } from "./theme";

/**
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
* The following assumptions have been made:
* - languageId < 256 => needs 8 bits
* - unique color count < 512 => needs 9 bits
*
* The binary format is:
* - -------------------------------------------
* 3322 2222 2222 1111 1111 1100 0000 0000
* 1098 7654 3210 9876 5432 1098 7654 3210
* - -------------------------------------------
* xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
* bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL
* - -------------------------------------------
* - L = LanguageId (8 bits)
* - T = StandardTokenType (2 bits)
* - B = Balanced bracket (1 bit)
* - F = FontStyle (4 bits)
* - f = foreground color (9 bits)
* - b = background color (9 bits)
*/
export const enum MetadataConsts {
LANGUAGEID_MASK = 0b00000000000000000000000011111111,
TOKEN_TYPE_MASK = 0b00000000000000000000001100000000,
BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000,
FONT_STYLE_MASK = 0b00000000000000000111100000000000,
FOREGROUND_MASK = 0b00000000111111111000000000000000,
BACKGROUND_MASK = 0b11111111000000000000000000000000,

LANGUAGEID_OFFSET = 0,
TOKEN_TYPE_OFFSET = 8,
BALANCED_BRACKETS_OFFSET = 10,
FONT_STYLE_OFFSET = 11,
FOREGROUND_OFFSET = 15,
BACKGROUND_OFFSET = 24
}

export class StackElementMetadata {

public static toBinaryStr(metadata: number): string {
let r = metadata.toString(2);
while (r.length < 32) {
r = '0' + r;
}
return r;
}

public static printMetadata(metadata: number): void {
const languageId = StackElementMetadata.getLanguageId(metadata);
const tokenType = StackElementMetadata.getTokenType(metadata);
const fontStyle = StackElementMetadata.getFontStyle(metadata);
const foreground = StackElementMetadata.getForeground(metadata);
const background = StackElementMetadata.getBackground(metadata);

console.log({
languageId: languageId,
tokenType: tokenType,
fontStyle: fontStyle,
foreground: foreground,
background: background,
});
}

public static getLanguageId(metadata: number): number {
return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET;
}

public static getTokenType(metadata: number): StandardTokenType {
return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
}

public static containsBalancedBrackets(metadata: number): boolean {
return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0;
}

public static getFontStyle(metadata: number): number {
return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET;
}

public static getForeground(metadata: number): number {
return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET;
}

public static getBackground(metadata: number): number {
return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET;
}

/**
* Updates the fields in `metadata`.
* A value of `0`, `NotSet` or `null` indicates that the corresponding field should be left as is.
*/
public static set(metadata: number, languageId: number, tokenType: OptionalStandardTokenType, containsBalancedBrackets: boolean | null, fontStyle: FontStyle, foreground: number, background: number): number {
let _languageId = StackElementMetadata.getLanguageId(metadata);
let _tokenType = StackElementMetadata.getTokenType(metadata);
let _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets(metadata) ? 1 : 0;
let _fontStyle = StackElementMetadata.getFontStyle(metadata);
let _foreground = StackElementMetadata.getForeground(metadata);
let _background = StackElementMetadata.getBackground(metadata);

if (languageId !== 0) {
_languageId = languageId;
}
if (tokenType !== OptionalStandardTokenType.NotSet) {
_tokenType = fromOptionalTokenType(tokenType);
}
if (containsBalancedBrackets !== null) {
_containsBalancedBracketsBit = containsBalancedBrackets ? 1 : 0;
}
if (fontStyle !== FontStyle.NotSet) {
_fontStyle = fontStyle;
}
if (foreground !== 0) {
_foreground = foreground;
}
if (background !== 0) {
_background = background;
}

return (
(_languageId << MetadataConsts.LANGUAGEID_OFFSET)
| (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET)
| (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET)
| (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET)
| (_foreground << MetadataConsts.FOREGROUND_OFFSET)
| (_background << MetadataConsts.BACKGROUND_OFFSET)
) >>> 0;
}
}

export function toOptionalTokenType(standardType: StandardTokenType): OptionalStandardTokenType {
return standardType as any as OptionalStandardTokenType;
}

function fromOptionalTokenType(
standardType:
| OptionalStandardTokenType.Other
| OptionalStandardTokenType.Comment
| OptionalStandardTokenType.String
| OptionalStandardTokenType.RegEx
): StandardTokenType {
return standardType as any as StandardTokenType;
}

// Must have the same values as `StandardTokenType`!
export const enum OptionalStandardTokenType {
Other = 0,
Comment = 1,
String = 2,
RegEx = 3,
// Indicates that no token type is set.
NotSet = 8
}
Loading

0 comments on commit 9157c7f

Please sign in to comment.