Replace Token.size -> text

For #68.
no-context · Aug 9, 2017 · abba951 · abba951
1 parent 6de68be
commit abba951
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 14 deletions.
diff --git a/moo.js b/moo.js
@@ -352,15 +352,15 @@
     var match = this._eat(re)
     var i = this._getGroup(match)
 
-    var group, value
+    var group, text
     if (i === -1) {
       group = this.error
 
       // consume rest of buffer
-      value = buffer.slice(index)
+      text = buffer.slice(index)
 
     } else {
-      value = match[0] // i+1
+      text = match[0]
       group = this.groups[i]
     }
 
@@ -369,25 +369,26 @@
     if (group.lineBreaks) {
       var matchNL = /\n/g
       var nl = 1
-      if (value === '\n') {
+      if (text === '\n') {
         lineBreaks = 1
       } else {
-        while (matchNL.exec(value)) { lineBreaks++; nl = matchNL.lastIndex }
+        while (matchNL.exec(text)) { lineBreaks++; nl = matchNL.lastIndex }
       }
     }
 
-    var size = value.length
     var token = {
-      type: (group.getType && group.getType(value)) || group.tokenType,
-      value: (group.value && group.value(value)) || value,
+      type: (group.getType && group.getType(text)) || group.tokenType,
+      value: (group.value && group.value(text)) || text,
+      text: text,
       toString: tokenToString,
       offset: index,
-      size: size,
       lineBreaks: lineBreaks,
       line: this.line,
       col: this.col,
     }
+    // nb. adding more props to token object will make V8 sad!
 
+    var size = text.length
     this.index += size
     this.line += lineBreaks
     if (lineBreaks !== 0) {

diff --git a/test/test.js b/test/test.js
@@ -196,7 +196,7 @@ describe('value transforms', () => {
     })).toThrow("has capture groups")
   })
 
-  test('transform & report correct size', () => {
+  test('transform & keep original', () => {
     let lexer = moo.compile({
       fubar: {match: /fubar/, value: x => x.slice(2)},
       string: {match: /".*?"/, value: x => x.slice(1, -1)},
@@ -206,10 +206,10 @@ describe('value transforms', () => {
     })
     lexer.reset('fubar "yes" quxx moomoomoomoo')
     let tokens = lexAll(lexer).filter(t => t.type !== 'space')
-    expect(tokens.shift()).toMatchObject({ type: 'fubar', value: 'bar', size: 5 })
-    expect(tokens.shift()).toMatchObject({ type: 'string', value: 'yes', size: 5 })
-    expect(tokens.shift()).toMatchObject({ value: 'quxx', size: 4 })
-    expect(tokens.shift()).toMatchObject({ value: 'moomoo', size: 12 })
+    expect(tokens.shift()).toMatchObject({ type: 'fubar', text: 'fubar', value: 'bar' })
+    expect(tokens.shift()).toMatchObject({ type: 'string', text: '"yes"', value: 'yes' })
+    expect(tokens.shift()).toMatchObject({ value: 'quxx' })
+    expect(tokens.shift()).toMatchObject({ value: 'moomoo' })
   })
 
 })
@@ -650,6 +650,7 @@ describe('errors', () => {
     })
     lexer.reset('foo\nbar')
     expect(lexer.next()).toMatchObject({type: 'error', value: 'foo\nbar', lineBreaks: 1})
+    expect(lexer.save()).toMatchObject({line: 2})
     expect(lexer.next()).toBe(undefined) // consumes rest of input
   })