Add negative lookahead to tokenizer (#130)

* Add negative lookahead to tokenizer

* Update tokenizer.test.ts

removed console.log

* bump

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scim2-parse-filter",
-  "version": "0.2.8",
+  "version": "0.2.10",
   "description": "This is a fork https://www.npmjs.com/package/scim2-filter version 0.2.0 with bug correction.",
   "main": "lib/src/index.js",
   "directories": {

src/parser.ts

@@ -10,7 +10,7 @@ export type Token = {
 export function tokenizer(f: string): Token[] {
   const ret: Token[] = [];
   let rest = f;
-  const patterns = /^(?:(\s+)|(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?)|("(?:[^"]|\\.|\n)*")|([[()]|]\.?)|(\w[-\w._:\/%]*))/;
+  const patterns = /^(?:(\s+)|(-?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?(?![-\w._:\/\)\s]))|("(?:[^"]|\\.|\n)*")|([[()]|]\.?)|(\w[-\w._:\/%]*))/;
   let n;
   while ((n = patterns.exec(rest))) {
     if (n[1] || n[0].length === 0) {

test/parse.test.ts

@@ -1,5 +1,5 @@
 import { eq, op, pr, and, or, v } from "./test_util";
-import { Filter, parse } from "../src";
+import { Filter, parse, stringify } from "../src";
 import { assert } from "chai";
 
 // When modifying or adding to these tests,
@@ -267,4 +267,20 @@ describe('parse', () => {
         eq("name", "xxx"))
     );
   });
+
+  describe('attrPath start with number', () => {
+    test('064869bf-be25-466f-803d-004a0540574b eq "bjensen"', eq("064869bf-be25-466f-803d-004a0540574b", "bjensen"));
+
+    it('consistent parse and stringify', () => {
+      const f : Filter = {
+        op: 'eq',
+        attrPath: '064869bf-be25-466f-803d-004a0540574b',
+        compValue: 'bjensen'
+      }
+      const string = stringify(f)
+      const ff = parse(string)
+
+      assert.deepEqual(f, ff)
+    })
+  })
 });

test/stringify.test.ts

@@ -253,4 +253,12 @@ describe('stringify', () => {
       )
     );
   });
+
+  it('consistent stringify and parse', () => {
+    const text = '064869bf-be25-466f-803d-004a0540574b eq "bjensen"'
+    const f = parse(text)
+    const string = stringify(f)
+
+    assert.deepEqual(text, string)
+  })
 });

test/tokenizer.test.ts

@@ -32,6 +32,13 @@ describe("tokenizer", () => {
     );
   });
 
+  it("0Field1 eq -12", () => {
+    assert.deepEqual(
+      [tok("0Field1", "Word"), tok("eq", "Word"), tok("-12", "Number"), EOT],
+      tokenizer("0Field1 eq -12")
+    );
+  });
+
   it("sub-attribute after ValPath", () => {
     assert.deepEqual(
         tokenizer('emails[type eq "work"].value eq "user@example.com"'),