Skip to content
This repository has been archived by the owner on Dec 15, 2022. It is now read-only.

Tokenize empty lines correctly #101

Merged
merged 1 commit into from
Sep 23, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions spec/grammar-spec.coffee
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,13 @@ describe "Grammar tokenization", ->
expect(tokens[3].value).toBe '!'
expect(tokens[3].scopes).toEqual ['source.hello', 'suffix.hello', 'emphasis.hello']

describe '::tokenizeLines(text)', ->
describe 'when the text is empty', ->
it 'returns a single line with a single token which has the global scope', ->
grammar = registry.grammarForScopeName('source.coffee')
lines = grammar.tokenizeLines('')
expect(lines).toEqual [[{value: '', scopes: ['source.coffee']}]]

describe "::tokenizeLine(line, ruleStack)", ->
describe "when the entire line matches a single pattern with no capture groups", ->
it "returns a single token with the correct scope", ->
Expand Down
2 changes: 1 addition & 1 deletion src/grammar.coffee
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ class Grammar
previousRuleStackLength = ruleStack.length
previousPosition = position

break if position is stringWithNewLine.length
break if position > line.length

if tokenCount >= @getMaxTokensPerLine() - 1
truncatedLine = true
Expand Down