From 9b1ac1094815cf2c5edb964f6c3094c7b07a2ff5 Mon Sep 17 00:00:00 2001 From: Indrek Ardel Date: Sat, 23 Sep 2017 14:45:47 +0300 Subject: [PATCH] Tokenize empty lines correctly --- spec/grammar-spec.coffee | 7 +++++++ src/grammar.coffee | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/spec/grammar-spec.coffee b/spec/grammar-spec.coffee index a7f20a4..5f0e355 100644 --- a/spec/grammar-spec.coffee +++ b/spec/grammar-spec.coffee @@ -57,6 +57,13 @@ describe "Grammar tokenization", -> expect(tokens[3].value).toBe '!' expect(tokens[3].scopes).toEqual ['source.hello', 'suffix.hello', 'emphasis.hello'] + describe '::tokenizeLines(text)', -> + describe 'when the text is empty', -> + it 'returns a single line with a single token which has the global scope', -> + grammar = registry.grammarForScopeName('source.coffee') + lines = grammar.tokenizeLines('') + expect(lines).toEqual [[{value: '', scopes: ['source.coffee']}]] + describe "::tokenizeLine(line, ruleStack)", -> describe "when the entire line matches a single pattern with no capture groups", -> it "returns a single token with the correct scope", -> diff --git a/src/grammar.coffee b/src/grammar.coffee index a45d3d8..84858de 100644 --- a/src/grammar.coffee +++ b/src/grammar.coffee @@ -135,7 +135,7 @@ class Grammar previousRuleStackLength = ruleStack.length previousPosition = position - break if position is stringWithNewLine.length + break if position > line.length if tokenCount >= @getMaxTokensPerLine() - 1 truncatedLine = true