From 169e45aa868bc8923cfef61ed7809dc8d4332e95 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 16 Apr 2024 19:03:27 +0900 Subject: [PATCH 1/7] basic functions --- examples/gno.land/p/demo/json/buffer.gno | 143 ++++++++++++++++++ examples/gno.land/p/demo/json/buffer_test.gno | 36 +++++ examples/gno.land/p/demo/json/parser.gno | 48 +----- examples/gno.land/p/demo/json/parser_test.gno | 35 ----- 4 files changed, 182 insertions(+), 80 deletions(-) diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 23fb53fb0ea..c45d4876799 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -28,6 +28,10 @@ func newBuffer(data []byte) *buffer { } } +func (b *buffer) reset() { + b.last = GO +} + // first retrieves the first non-whitespace (or other escaped) character in the buffer. func (b *buffer) first() (byte, error) { for ; b.index < b.length; b.index++ { @@ -483,3 +487,142 @@ func numberKind2f64(value interface{}) (result float64, err error) { return } + +func (b *buffer) nextToken() (string, error) { + var token strings.Builder + + c, err := b.first() + if err != nil { + return "", err + } + + switch { + case isNumber(c): + token.WriteByte(c) + b.next() + + for b.index < b.length && (isNumber(b.data[b.index]) || b.data[b.index] == dot) { + token.WriteByte(b.data[b.index]) + b.next() + } + case isAlpha(c): + for b.index < b.length && isAlpha(b.data[b.index]) { + token.WriteByte(b.data[b.index]) + b.next() + } + case isQuote(c): + quote := c + token.WriteByte(c) + b.index++ + for b.index < b.length && b.data[b.index] != quote { + token.WriteByte(b.data[b.index]) + b.index++ + } + if b.index < b.length { + token.WriteByte(b.data[b.index]) + b.index++ + } + case isVariable(c): + token.WriteByte(c) + b.index++ + for b.index < b.length && isAlphaNumeric(b.data[b.index]) { + token.WriteByte(b.data[b.index]) + b.index++ + } + default: + token.WriteByte(c) + b.next() + } + + return token.String(), nil +} + +func isAlpha(c byte) bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') +} + +func isAlphaNumeric(c byte) bool { + return isAlpha(c) || isNumber(c) || c == '_' +} + +func isQuote(c byte) bool { + return c == doubleQuote || c == singleQuote +} + +func isVariable(c byte) bool { + return c == '@' || c == '$' +} + +var ( + ErrInvalidSymbol = errors.New("invalid symbol") + ErrMissingLeftParen = errors.New("missing left parenthesis") + ErrInvalidOperationFunc = errors.New("invalid operation or function") +) + +var binOps = map[string]int { + "+": 1, + "-": 1, + "*": 2, + "/": 2, + "^": 3, + "%": 2, +} + +func isOperation(s string) bool { + _, ok := binOps[s] + return ok +} + +func operationPriority(op string) int { + if p, ok := binOps[op]; ok { + return p + } + + return 0 +} + +var reservedFunc = map[string]bool{ + "add": true, +} + +func (b *buffer) processOperation(token string, stack *[]string, result *[]string) error { + for len(*stack) > 0 { + top := (*stack)[len(*stack)-1] + if isFunction(top) || (isOperation(top) && operationPriority(top) >= operationPriority(token)) { + *result = append(*result, top) + *stack = (*stack)[:len(*stack)-1] + } else { + break + } + } + *stack = append(*stack, token) + return nil +} + +func isFunction(s string) bool { + return reservedFunc[s] +} + +// func (b *buffer) rpn() ([]string, error) { +// var ( +// stack, result []string +// token string +// ) + +// for { +// b.reset() + +// token, err := b.nextToken() +// if err != nil { +// if err == io.EOF { +// break +// } +// return nil, err +// } + +// switch { +// case isOperator(token): +// if err := b. +// } +// } +// } \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index a1acce4eba0..bedcb9bd8bb 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -2,6 +2,7 @@ package json import ( "testing" + "io" ) func TestBufferCurrent(t *testing.T) { @@ -622,3 +623,38 @@ func TestBufferFirst(t *testing.T) { }) } } + +func TestNextToken(t *testing.T) { + testCases := []struct { + input string + expected []string + }{ + {"1 + 2", []string{"1", "+", "2"}}, + {"(1 + 2) * 3", []string{"(", "1", "+", "2", ")", "*", "3"}}, + {"sin(0)", []string{"sin", "(", "0", ")"}}, + {"max(1, 2, 3)", []string{"max", "(", "1", ",", "2", ",", "3", ")"}}, + {"2 * (3 + 4)", []string{"2", "*", "(", "3", "+", "4", ")"}}, + {"2 * (3 + 4) / 5", []string{"2", "*", "(", "3", "+", "4", ")", "/", "5"}}, + {"2 * @length", []string{"2", "*", "@length"}}, + {"$.price * 1.5", []string{"$.price", "*", "1.5"}}, + {"-1e6", []string{"-1e6"}}, + {"'hello'", []string{"'hello'"}}, + {"true", []string{"true"}}, + {"PI", []string{"PI"}}, + } + + for _, tc := range testCases { + b := &buffer{data: []byte(tc.input)} + var result []string + for { + token, err := b.nextToken() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("Unexpected error: %v", err) + } + result = append(result, token) + } + } +} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 9a2c3a8c817..72e09e6b916 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -82,48 +82,6 @@ func ParseFloatLiteral(bytes []byte) (float64, error) { return f, nil } -func ParseIntLiteral(bytes []byte) (int64, error) { - if len(bytes) == 0 { - return 0, errors.New("JSON Error: empty byte slice found while parsing integer value") - } - - neg, bytes := trimNegativeSign(bytes) - - var n uint64 = 0 - for _, c := range bytes { - if notDigit(c) { - return 0, errors.New("JSON Error: non-digit characters found while parsing integer value") - } - - if n > maxUint64/10 { - return 0, errors.New("JSON Error: numeric value exceeds the range limit") - } - - n *= 10 - - n1 := n + uint64(c-'0') - if n1 < n { - return 0, errors.New("JSON Error: numeric value exceeds the range limit") - } - - n = n1 - } - - if n > maxInt64 { - if neg && n == absMinInt64 { - return -absMinInt64, nil - } - - return 0, errors.New("JSON Error: numeric value exceeds the range limit") - } - - if neg { - return -int64(n), nil - } - - return int64(n), nil -} - // extractMantissaAndExp10 parses a byte slice representing a decimal number and extracts the mantissa and the exponent of its base-10 representation. // It iterates through the bytes, constructing the mantissa by treating each byte as a digit. // If a decimal point is encountered, the function keeps track of the position of the decimal point to calculate the exponent. @@ -147,7 +105,7 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { continue } - if notDigit(c) { + if !isNumber(c) { return 0, 0, errors.New("JSON Error: non-digit characters found while parsing integer value") } @@ -175,8 +133,8 @@ func trimNegativeSign(bytes []byte) (bool, []byte) { return false, bytes } -func notDigit(c byte) bool { - return (c & 0xF0) != 0x30 +func isNumber(c byte) bool { + return (c & 0xF0) == 0x30 } // lower converts a byte to lower case if it is an uppercase letter. diff --git a/examples/gno.land/p/demo/json/parser_test.gno b/examples/gno.land/p/demo/json/parser_test.gno index 44a2fee6404..105ecb6004f 100644 --- a/examples/gno.land/p/demo/json/parser_test.gno +++ b/examples/gno.land/p/demo/json/parser_test.gno @@ -154,38 +154,3 @@ func TestParseFloat_May_Interoperability_Problem(t *testing.T) { }) } } - -func TestParseIntLiteral(t *testing.T) { - tests := []struct { - input string - expected int64 - }{ - {"0", 0}, - {"1", 1}, - {"-1", -1}, - {"12345", 12345}, - {"-12345", -12345}, - {"9223372036854775807", 9223372036854775807}, - {"-9223372036854775808", -9223372036854775808}, - {"-92233720368547758081", 0}, - {"18446744073709551616", 0}, - {"9223372036854775808", 0}, - {"-9223372036854775809", 0}, - {"", 0}, - {"abc", 0}, - {"12345x", 0}, - {"123e5", 0}, - {"9223372036854775807x", 0}, - {"27670116110564327410", 0}, - {"-27670116110564327410", 0}, - } - - for _, tt := range tests { - t.Run(tt.input, func(t *testing.T) { - got, _ := ParseIntLiteral([]byte(tt.input)) - if got != tt.expected { - t.Errorf("ParseIntLiteral(%s): got %v, want %v", tt.input, got, tt.expected) - } - }) - } -} From 81edb071136fbb355fa94590ec8887bd504bdfa1 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 17 Apr 2024 21:02:33 +0900 Subject: [PATCH 2/7] path tokenizer --- examples/gno.land/p/demo/json/buffer.gno | 260 +-------------- examples/gno.land/p/demo/json/buffer_test.gno | 37 +-- examples/gno.land/p/demo/json/node.gno | 35 ++ examples/gno.land/p/demo/json/node_test.gno | 227 +++++++++++-- examples/gno.land/p/demo/json/parser.gno | 11 +- examples/gno.land/p/demo/json/path.gno | 305 ++++++++++++++++++ examples/gno.land/p/demo/json/path_test.gno | 70 +++- examples/gno.land/p/demo/json/token.gno | 2 +- 8 files changed, 621 insertions(+), 326 deletions(-) diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index c45d4876799..8d77d2d9d2d 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -179,9 +179,9 @@ var significantTokens = map[byte]bool{ // filterTokens stores the filter expression tokens. var filterTokens = map[byte]bool{ - aesterisk: true, // wildcard - andSign: true, - orSign: true, + asterisk: true, // wildcard + andSign: true, + orSign: true, } // skipToNextSignificantToken advances the buffer index to the next significant character. @@ -223,121 +223,6 @@ func (b *buffer) backslash() bool { return count%2 != 0 } -// numIndex holds a map of valid numeric characters -var numIndex = map[byte]bool{ - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - '.': true, - 'e': true, - 'E': true, -} - -// pathToken checks if the current token is a valid JSON path token. -func (b *buffer) pathToken() error { - var stack []byte - - inToken := false - inNumber := false - first := b.index - - for b.index < b.length { - c := b.data[b.index] - - switch { - case c == doubleQuote || c == singleQuote: - inToken = true - if err := b.step(); err != nil { - return errors.New("error stepping through buffer") - } - - if err := b.skip(c); err != nil { - return errors.New("unmatched quote in path") - } - - if b.index >= b.length { - return errors.New("unmatched quote in path") - } - - case c == bracketOpen || c == parenOpen: - inToken = true - stack = append(stack, c) - - case c == bracketClose || c == parenClose: - inToken = true - if len(stack) == 0 || (c == bracketClose && stack[len(stack)-1] != bracketOpen) || (c == parenClose && stack[len(stack)-1] != parenOpen) { - return errors.New("mismatched bracket or parenthesis") - } - - stack = stack[:len(stack)-1] - - case pathStateContainsValidPathToken(c): - inToken = true - - case c == plus || c == minus: - if inNumber || (b.index > 0 && numIndex[b.data[b.index-1]]) { - inToken = true - } else if !inToken && (b.index+1 < b.length && numIndex[b.data[b.index+1]]) { - inToken = true - inNumber = true - } else if !inToken { - return errors.New("unexpected operator at start of token") - } - - default: - if len(stack) != 0 || inToken { - inToken = true - } else { - goto end - } - } - - b.index++ - } - -end: - if len(stack) != 0 { - return errors.New("unclosed bracket or parenthesis at end of path") - } - - if first == b.index { - return errors.New("no token found") - } - - if inNumber && !numIndex[b.data[b.index-1]] { - inNumber = false - } - - return nil -} - -func pathStateContainsValidPathToken(c byte) bool { - if _, ok := significantTokens[c]; ok { - return true - } - - if _, ok := filterTokens[c]; ok { - return true - } - - if _, ok := numIndex[c]; ok { - return true - } - - if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' { - return true - } - - return false -} - func (b *buffer) numeric(token bool) error { if token { b.last = GO @@ -487,142 +372,3 @@ func numberKind2f64(value interface{}) (result float64, err error) { return } - -func (b *buffer) nextToken() (string, error) { - var token strings.Builder - - c, err := b.first() - if err != nil { - return "", err - } - - switch { - case isNumber(c): - token.WriteByte(c) - b.next() - - for b.index < b.length && (isNumber(b.data[b.index]) || b.data[b.index] == dot) { - token.WriteByte(b.data[b.index]) - b.next() - } - case isAlpha(c): - for b.index < b.length && isAlpha(b.data[b.index]) { - token.WriteByte(b.data[b.index]) - b.next() - } - case isQuote(c): - quote := c - token.WriteByte(c) - b.index++ - for b.index < b.length && b.data[b.index] != quote { - token.WriteByte(b.data[b.index]) - b.index++ - } - if b.index < b.length { - token.WriteByte(b.data[b.index]) - b.index++ - } - case isVariable(c): - token.WriteByte(c) - b.index++ - for b.index < b.length && isAlphaNumeric(b.data[b.index]) { - token.WriteByte(b.data[b.index]) - b.index++ - } - default: - token.WriteByte(c) - b.next() - } - - return token.String(), nil -} - -func isAlpha(c byte) bool { - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') -} - -func isAlphaNumeric(c byte) bool { - return isAlpha(c) || isNumber(c) || c == '_' -} - -func isQuote(c byte) bool { - return c == doubleQuote || c == singleQuote -} - -func isVariable(c byte) bool { - return c == '@' || c == '$' -} - -var ( - ErrInvalidSymbol = errors.New("invalid symbol") - ErrMissingLeftParen = errors.New("missing left parenthesis") - ErrInvalidOperationFunc = errors.New("invalid operation or function") -) - -var binOps = map[string]int { - "+": 1, - "-": 1, - "*": 2, - "/": 2, - "^": 3, - "%": 2, -} - -func isOperation(s string) bool { - _, ok := binOps[s] - return ok -} - -func operationPriority(op string) int { - if p, ok := binOps[op]; ok { - return p - } - - return 0 -} - -var reservedFunc = map[string]bool{ - "add": true, -} - -func (b *buffer) processOperation(token string, stack *[]string, result *[]string) error { - for len(*stack) > 0 { - top := (*stack)[len(*stack)-1] - if isFunction(top) || (isOperation(top) && operationPriority(top) >= operationPriority(token)) { - *result = append(*result, top) - *stack = (*stack)[:len(*stack)-1] - } else { - break - } - } - *stack = append(*stack, token) - return nil -} - -func isFunction(s string) bool { - return reservedFunc[s] -} - -// func (b *buffer) rpn() ([]string, error) { -// var ( -// stack, result []string -// token string -// ) - -// for { -// b.reset() - -// token, err := b.nextToken() -// if err != nil { -// if err == io.EOF { -// break -// } -// return nil, err -// } - -// switch { -// case isOperator(token): -// if err := b. -// } -// } -// } \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/buffer_test.gno b/examples/gno.land/p/demo/json/buffer_test.gno index bedcb9bd8bb..2569385197e 100644 --- a/examples/gno.land/p/demo/json/buffer_test.gno +++ b/examples/gno.land/p/demo/json/buffer_test.gno @@ -1,8 +1,8 @@ package json import ( - "testing" "io" + "testing" ) func TestBufferCurrent(t *testing.T) { @@ -623,38 +623,3 @@ func TestBufferFirst(t *testing.T) { }) } } - -func TestNextToken(t *testing.T) { - testCases := []struct { - input string - expected []string - }{ - {"1 + 2", []string{"1", "+", "2"}}, - {"(1 + 2) * 3", []string{"(", "1", "+", "2", ")", "*", "3"}}, - {"sin(0)", []string{"sin", "(", "0", ")"}}, - {"max(1, 2, 3)", []string{"max", "(", "1", ",", "2", ",", "3", ")"}}, - {"2 * (3 + 4)", []string{"2", "*", "(", "3", "+", "4", ")"}}, - {"2 * (3 + 4) / 5", []string{"2", "*", "(", "3", "+", "4", ")", "/", "5"}}, - {"2 * @length", []string{"2", "*", "@length"}}, - {"$.price * 1.5", []string{"$.price", "*", "1.5"}}, - {"-1e6", []string{"-1e6"}}, - {"'hello'", []string{"'hello'"}}, - {"true", []string{"true"}}, - {"PI", []string{"PI"}}, - } - - for _, tc := range testCases { - b := &buffer{data: []byte(tc.input)} - var result []string - for { - token, err := b.nextToken() - if err != nil { - if err == io.EOF { - break - } - t.Fatalf("Unexpected error: %v", err) - } - result = append(result, token) - } - } -} \ No newline at end of file diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 1e71a101e62..05bba2127a6 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -56,6 +56,23 @@ func NewNode(prev *Node, b *buffer, typ ValueType, key **string) (*Node, error) return curr, nil } +func valueNode(prev *Node, key string, typ ValueType, value interface{}) *Node { + curr := &Node{ + prev: prev, + data: nil, + borders: [2]int{0, 0}, + key: &key, + nodeType: typ, + modified: false, + } + + if curr.value != nil { + curr.value = value + } + + return curr +} + // load retrieves the value of the current node. func (n *Node) load() interface{} { return n.value @@ -480,31 +497,49 @@ func ObjectNode(key string, value map[string]*Node) *Node { // IsArray returns true if the current node is array type. func (n *Node) IsArray() bool { + if n == nil { + return false + } return n.nodeType == Array } // IsObject returns true if the current node is object type. func (n *Node) IsObject() bool { + if n == nil { + return false + } return n.nodeType == Object } // IsNull returns true if the current node is null type. func (n *Node) IsNull() bool { + if n == nil { + return false + } return n.nodeType == Null } // IsBool returns true if the current node is boolean type. func (n *Node) IsBool() bool { + if n == nil { + return false + } return n.nodeType == Boolean } // IsString returns true if the current node is string type. func (n *Node) IsString() bool { + if n == nil { + return false + } return n.nodeType == String } // IsNumber returns true if the current node is number type. func (n *Node) IsNumber() bool { + if n == nil { + return false + } return n.nodeType == Number } diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index dbc82369f68..e129476f041 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -300,40 +300,227 @@ func TestNode_GetBool_Fail(t *testing.T) { } } -func TestNode_IsBool(t *testing.T) { - tests := []simpleNode{ - {"true", BoolNode("", true)}, - {"false", BoolNode("", false)}, +func TestNode_IsBool_With_Unmarshal(t *testing.T) { + tests := []struct { + name string + json []byte + want bool + }{ + {"true", []byte("true"), true}, + {"false", []byte("false"), true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if !tt.node.IsBool() { + root, err := Unmarshal(tt.json) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err.Error()) + } + + if root.IsBool() != tt.want { t.Errorf("%s should be a bool", tt.name) } }) } } -func TestNode_IsBool_With_Unmarshal(t *testing.T) { +func TestNode_IsString(t *testing.T) { tests := []struct { name string - json []byte + node *Node want bool }{ - {"true", []byte("true"), true}, - {"false", []byte("false"), true}, + { + name: "String node", + node: &Node{nodeType: String}, + want: true, + }, + { + name: "Non-string node", + node: &Node{nodeType: Number}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - root, err := Unmarshal(tt.json) - if err != nil { - t.Errorf("Error on Unmarshal(): %s", err.Error()) + if got := tt.node.IsString(); got != tt.want { + t.Errorf("Node.IsString() = %v, want %v", got, tt.want) } + }) + } +} - if root.IsBool() != tt.want { - t.Errorf("%s should be a bool", tt.name) +func TestNode_IsNumber(t *testing.T) { + tests := []struct { + name string + node *Node + want bool + }{ + { + name: "Number node", + node: &Node{nodeType: Number}, + want: true, + }, + { + name: "Non-number node", + node: &Node{nodeType: String}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.IsNumber(); got != tt.want { + t.Errorf("Node.IsNumber() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNode_IsBool(t *testing.T) { + tests := []struct { + name string + node *Node + want bool + }{ + { + name: "Bool node", + node: &Node{nodeType: Boolean}, + want: true, + }, + { + name: "Non-bool node", + node: &Node{nodeType: String}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.IsBool(); got != tt.want { + t.Errorf("Node.IsBool() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNode_IsNull(t *testing.T) { + tests := []struct { + name string + node *Node + want bool + }{ + { + name: "Null node", + node: &Node{nodeType: Null}, + want: true, + }, + { + name: "Non-null node", + node: &Node{nodeType: String}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.IsNull(); got != tt.want { + t.Errorf("Node.IsNull() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNode_IsArray(t *testing.T) { + tests := []struct { + name string + node *Node + want bool + }{ + { + name: "Array node", + node: &Node{nodeType: Array}, + want: true, + }, + { + name: "Non-array node", + node: &Node{nodeType: String}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.IsArray(); got != tt.want { + t.Errorf("Node.IsArray() = %v, want %v", got, tt.want) + } + }) + } + + root, err := Unmarshal(sampleArr) + if err != nil { + t.Errorf("Error on Unmarshal(): %s", err) + return + } + + if root.Type() != Array { + t.Errorf(ufmt.Sprintf("Must be an array. got: %s", root.Type().String())) + } +} + +func TestNode_IsObject(t *testing.T) { + tests := []struct { + name string + node *Node + want bool + }{ + { + name: "Object node", + node: &Node{nodeType: Object}, + want: true, + }, + { + name: "Non-object node", + node: &Node{nodeType: String}, + want: false, + }, + { + name: "Nil node", + node: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.node.IsObject(); got != tt.want { + t.Errorf("Node.IsObject() = %v, want %v", got, tt.want) } }) } @@ -593,18 +780,6 @@ func TestNode_GetArray_Fail(t *testing.T) { } } -func TestNode_IsArray(t *testing.T) { - root, err := Unmarshal(sampleArr) - if err != nil { - t.Errorf("Error on Unmarshal(): %s", err) - return - } - - if root.Type() != Array { - t.Errorf(ufmt.Sprintf("Must be an array. got: %s", root.Type().String())) - } -} - func TestNode_ArrayEach(t *testing.T) { tests := []struct { name string diff --git a/examples/gno.land/p/demo/json/parser.gno b/examples/gno.land/p/demo/json/parser.gno index 72e09e6b916..4efd9484e89 100644 --- a/examples/gno.land/p/demo/json/parser.gno +++ b/examples/gno.land/p/demo/json/parser.gno @@ -4,6 +4,7 @@ import ( "bytes" "errors" "strconv" + "unicode" el "gno.land/p/demo/json/eisel_lemire" ) @@ -105,7 +106,7 @@ func extractMantissaAndExp10(bytes []byte) (uint64, int, error) { continue } - if !isNumber(c) { + if !unicode.IsDigit(rune(c)) { return 0, 0, errors.New("JSON Error: non-digit characters found while parsing integer value") } @@ -133,11 +134,11 @@ func trimNegativeSign(bytes []byte) (bool, []byte) { return false, bytes } -func isNumber(c byte) bool { - return (c & 0xF0) == 0x30 -} - // lower converts a byte to lower case if it is an uppercase letter. func lower(c byte) byte { return c | 0x20 } + +func isAlphaNumeric(c byte) bool { + return unicode.IsLetter(rune(c)) || unicode.IsDigit(rune(c)) +} diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 31f7e04633f..93e13c19e0d 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -1,7 +1,12 @@ package json +// ref: https://support.smartbear.com/alertsite/docs/monitors/api/endpoint/jsonpath.html + import ( "errors" + "io" + "strings" + "unicode" ) // ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. @@ -76,3 +81,303 @@ func extractNextSegment(buf *buffer, result *[]string) { *result = append(*result, segment) } } + +// numIndex holds a map of valid numeric characters +var numIndex = map[byte]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '.': true, + 'e': true, + 'E': true, +} + +// pathToken checks if the current token is a valid JSON path token. +func (b *buffer) pathToken() error { + var ( + stack []byte + inToken, inNumber bool + ) + + first := b.index + + for b.index < b.length { + c := b.data[b.index] + + switch { + case c == doubleQuote, c == singleQuote: + inToken = true + if err := b.step(); err != nil { + return errors.New("error stepping through buffer") + } + + if err := b.skip(c); err != nil { + return errors.New("unmatched quote in path") + } + + if b.index >= b.length { + return errors.New("unmatched quote in path") + } + + case c == bracketOpen, c == parenOpen: + inToken = true + stack = append(stack, c) + + case c == bracketClose, c == parenClose: + inToken = true + ls := len(stack) + if ls == 0 { + return errors.New("unexpected end of path") + } + + if (c == bracketClose && stack[ls-1] != bracketOpen) || (c == parenClose && stack[ls-1] != parenOpen) { + return errors.New("mismatched bracket or parenthesis") + } + + stack = stack[:ls-1] + + case pathStateContainsValidPathToken(c): + inToken = true + + case c == plus, c == minus: + if inNumber || (b.index > 0 && numIndex[b.data[b.index-1]]) { + inToken = true + } + + if !inToken && (b.index+1 < b.length && numIndex[b.data[b.index+1]]) { + inToken = true + inNumber = true + } + + if !inToken { + return errors.New("unexpected operator at start of token") + } + + default: + if len(stack) != 0 || inToken { + inToken = true + } else { + goto end + } + } + + b.index++ + } + +end: + if len(stack) != 0 { + return errors.New("unclosed bracket or parenthesis at end of path") + } + + if first == b.index { + return errors.New("no token found") + } + + if inNumber && !numIndex[b.data[b.index-1]] { + inNumber = false + } + + return nil +} + +func pathStateContainsValidPathToken(c byte) bool { + if _, ok := significantTokens[c]; ok { + return true + } + + if _, ok := filterTokens[c]; ok { + return true + } + + if _, ok := numIndex[c]; ok { + return true + } + + if unicode.IsLetter(rune(c)) { + return true + } + + return false +} + +func (b *buffer) pathTokenizer() ([]string, error) { + var result []string + + for { + b.reset() + c, err := b.first() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + var token string + switch { + case unicode.IsDigit(rune(c)), c == dot: + start := b.index + err = b.numeric(true) + if err == io.EOF { + token = string(b.sliceFromIndices(start, b.index)) + } else if err != nil { + if c == dot { + token = "." + b.index = start + } else { + return nil, err + } + } else { + token = string(b.sliceFromIndices(start, b.index)) + b.index-- + } + + case c == singleQuote, c == doubleQuote: + start := b.index + err = b.string(c, true) + if err != nil { + return nil, errors.New("error stepping through buffer") + } + token = string(b.sliceFromIndices(start, b.index+1)) + + case c == dollarSign, c == atSign: + start := b.index + err = b.findNextToken() + if err != nil && err != io.EOF { + return nil, err + } + token = string(b.sliceFromIndices(start, b.index)) + if err != io.EOF { + b.index-- + } + + case c == parenOpen, c == parenClose: + token = string(c) + + default: + start := b.index + for ; b.index < b.length; b.index++ { + c = b.data[b.index] + if c == parenOpen { + break + } + + rc := rune(c) + if unicode.IsLetter(rc) && !unicode.IsDigit(rc) && c != '_' { + break + } + } + err = b.step() + if err != nil && err != io.EOF { + return nil, err + } + + slice := string(b.sliceFromIndices(start, b.index)) + token = strings.ToLower(slice) + + if err != io.EOF { + b.index-- + } + } + + result = append(result, token) + err = b.step() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + } + + return result, nil +} + +func (b *buffer) findNextToken() error { + var ( + c byte + stack []byte + find bool + start int + ) + + for b.index < b.length { + c = b.data[b.index] + + switch { + case c == singleQuote, c == doubleQuote: + find = true + if err := b.step(); err != nil { + return errors.New("error stepping through buffer") + } + if err := b.skip(c); err == io.EOF { + return errors.New("unmatched quote in path") + } + case c == bracketOpen, c == parenOpen: + find = true + stack = append(stack, c) + case c == bracketClose, c == parenClose: + find = true + ls := len(stack) + if ls == 0 { + if start == b.index { + return errors.New("unexpected end of path") + } + break + } + expected := bracketOpen + if c == parenClose { + expected = parenOpen + } + if stack[ls-1] != expected { + return errors.New("mismatched bracket or parenthesis") + } + stack = stack[:ls-1] + case c == dot, c == atSign, c == dollarSign, c == question, c == asterisk, isAlphaNumeric(c): + find = true + b.index++ + continue + case len(stack) != 0: + find = true + b.index++ + continue + case c == minus, c == plus: + if !find { + find = true + start = b.index + if err := b.numeric(true); err == nil || err == io.EOF { + b.index-- + b.index++ + continue + } + b.index = start + } + fallthrough + default: + break + } + + if len(stack) == 0 { + break + } + b.index++ + } + + if len(stack) != 0 { + return errors.New("unclosed bracket or parenthesis at end of path") + } + if start == b.index { + return b.step() + } + if b.index >= b.length { + return io.EOF + } + return nil +} diff --git a/examples/gno.land/p/demo/json/path_test.gno b/examples/gno.land/p/demo/json/path_test.gno index dd242849f03..eb175c4ee82 100644 --- a/examples/gno.land/p/demo/json/path_test.gno +++ b/examples/gno.land/p/demo/json/path_test.gno @@ -4,7 +4,7 @@ import ( "testing" ) -func TestParseJSONPath(t *testing.T) { +func TestPath_ParseJSONPath(t *testing.T) { tests := []struct { name string path string @@ -49,6 +49,74 @@ func TestParseJSONPath(t *testing.T) { } } +func TestPath_pathTokenizer(t *testing.T) { + tests := []struct { + name string + want []string + isErr bool + }{ + { + name: "", + want: []string{}, + }, + { + name: "1234", + want: []string{"1234"}, + }, + { + name: "12e34", + want: []string{"12e34"}, + }, + { + name: "12.34", + want: []string{"12.34"}, + }, + { + name: "'foo'", + want: []string{"'foo'"}, + }, + { + name: "@.length", + want: []string{"@.length"}, + }, + { + name: "$.var2", + want: []string{"$.var2"}, + }, + { + name: "(@.length)", + want: []string{"(", "@.length", ")"}, + }, + { + name: `"{"`, + want: []string{`"{"`}, + }, + { + name: "@.[", + isErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := &buffer{ + data: []byte(tt.name), + length: len(tt.name), + } + + got, err := b.pathTokenizer() + if (err != nil) != tt.isErr { + t.Errorf("pathTokenizer() error = %v, wantErr %v", err, tt.isErr) + return + } + + if !isEqualSlice(got, tt.want) { + t.Errorf("pathTokenizer() got = %v, want %v", got, tt.want) + } + }) + } +} + func isEqualSlice(a, b []string) bool { if len(a) != len(b) { return false diff --git a/examples/gno.land/p/demo/json/token.gno b/examples/gno.land/p/demo/json/token.gno index 4791850bf46..85776d3d227 100644 --- a/examples/gno.land/p/demo/json/token.gno +++ b/examples/gno.land/p/demo/json/token.gno @@ -17,7 +17,7 @@ const ( whiteSpace = ' ' plus = '+' minus = '-' - aesterisk = '*' + asterisk = '*' bang = '!' question = '?' newLine = '\n' From 82e5563e8e40b44e418599a926d1a8759d79ee31 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 17 Apr 2024 21:06:01 +0900 Subject: [PATCH 3/7] fix: type --- examples/gno.land/p/demo/json/path.gno | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 93e13c19e0d..15900476c52 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -336,7 +336,7 @@ func (b *buffer) findNextToken() error { if c == parenClose { expected = parenOpen } - if stack[ls-1] != expected { + if rune(stack[ls-1]) != expected { return errors.New("mismatched bracket or parenthesis") } stack = stack[:ls-1] From 9a0c75b9c973d4d8be9e15312d1fce9f3bde39e0 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Tue, 4 Jun 2024 17:43:04 +0900 Subject: [PATCH 4/7] feat: basic expr handlers --- examples/gno.land/p/demo/json/buffer.gno | 129 +++- examples/gno.land/p/demo/json/node.gno | 76 +++ examples/gno.land/p/demo/json/node_test.gno | 49 +- examples/gno.land/p/demo/json/path.gno | 707 ++++++++++++-------- examples/gno.land/p/demo/json/path_test.gno | 280 ++++++-- 5 files changed, 831 insertions(+), 410 deletions(-) diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index 8d77d2d9d2d..fd459ed69ed 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -126,16 +126,7 @@ func (b *buffer) skipAny(endTokens map[byte]bool) error { b.index++ } - // build error message - var tokens []string - for token := range endTokens { - tokens = append(tokens, string(token)) - } - - return ufmt.Errorf( - "EOF reached before encountering one of the expected tokens: %s", - strings.Join(tokens, ", "), - ) + return io.EOF } // skipAndReturnIndex moves the buffer index forward by one and returns the new index. @@ -372,3 +363,121 @@ func numberKind2f64(value interface{}) (result float64, err error) { return } + +// numIndex holds a map of valid numeric characters +var numIndex = map[byte]bool{ + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + '.': true, + 'e': true, + 'E': true, +} + +// pathToken checks if the current token is a valid JSON path token. +func (b *buffer) pathToken() error { + var stack []byte + + inToken := false + inNumber := false + first := b.index + + for b.index < b.length { + c := b.data[b.index] + + switch { + case c == singleQuote: + fallthrough + + case c == doubleQuote: + inToken = true + if err := b.step(); err != nil { + return errors.New("error stepping through buffer") + } + + if err := b.skip(c); err != nil { + return errors.New("unmatched quote in path") + } + + if b.index >= b.length { + return errors.New("unmatched quote in path") + } + + case c == bracketOpen || c == parenOpen: + inToken = true + stack = append(stack, c) + + case c == bracketClose || c == parenClose: + inToken = true + if len(stack) == 0 || (c == bracketClose && stack[len(stack)-1] != bracketOpen) || (c == parenClose && stack[len(stack)-1] != parenOpen) { + return errors.New("mismatched bracket or parenthesis") + } + + stack = stack[:len(stack)-1] + + case pathStateContainsValidPathToken(c): + inToken = true + + case c == plus || c == minus: + if inNumber || (b.index > 0 && numIndex[b.data[b.index-1]]) { + inToken = true + } else if !inToken && (b.index+1 < b.length && numIndex[b.data[b.index+1]]) { + inToken = true + inNumber = true + } else if !inToken { + return errors.New("unexpected operator at start of token") + } + + default: + if len(stack) != 0 || inToken { + inToken = true + } else { + goto end + } + } + + b.index++ + } + +end: + if len(stack) != 0 { + return errors.New("unclosed bracket or parenthesis at end of path") + } + + if first == b.index { + return errors.New("no token found") + } + + if inNumber && !numIndex[b.data[b.index-1]] { + inNumber = false + } + + return nil +} + +func pathStateContainsValidPathToken(c byte) bool { + if _, ok := significantTokens[c]; ok { + return true + } + + if _, ok := filterTokens[c]; ok { + return true + } + + if _, ok := numIndex[c]; ok { + return true + } + + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' { + return true + } + + return false +} diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index 05bba2127a6..cd4cd45cc0f 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -1,7 +1,9 @@ package json import ( + "bytes" "errors" + "sort" "strconv" "strings" @@ -1116,3 +1118,77 @@ func Must(root *Node, expect error) *Node { return root } + +func (n *Node) Keys() []string { + if n == nil { + return nil + } + result := make([]string, 0, len(n.next)) + for key := range n.next { + result = append(result, key) + } + return result +} + +type nodeKeys []string + +func (keys nodeKeys) Len() int { + return len(keys) +} + +func (keys nodeKeys) Less(i, j int) bool { + return keys[i] < keys[j] +} + +func (keys nodeKeys) Swap(i, j int) { + keys[i], keys[j] = keys[j], keys[i] +} + +func (n *Node) getSortedChildren() (result []*Node) { + if n == nil { + return nil + } + + size := len(n.next) + if n.IsObject() { + result = make([]*Node, size) + keys := nodeKeys(n.Keys()) + sort.Sort(keys) + + for i, key := range keys { + result[i] = n.next[key] + } + } else if n.IsArray() { + result = make([]*Node, size) + for _, elem := range n.next { + result[*elem.index] = elem + } + } + + return result +} + +func deepEqual(n1, n2 *Node) bool { + if n1 == nil && n2 == nil { + return true + } + if n1 == nil || n2 == nil { + return false + } + if n1.nodeType != n2.nodeType { + return false + } + if n1.value != n2.value { + return false + } + if len(n1.next) != len(n2.next) { + return false + } + for key, child1 := range n1.next { + child2, ok := n2.next[key] + if !ok || !deepEqual(child1, child2) { + return false + } + } + return true +} diff --git a/examples/gno.land/p/demo/json/node_test.gno b/examples/gno.land/p/demo/json/node_test.gno index e129476f041..819923bd4f7 100644 --- a/examples/gno.land/p/demo/json/node_test.gno +++ b/examples/gno.land/p/demo/json/node_test.gno @@ -1,7 +1,6 @@ package json import ( - "bytes" "sort" "strconv" "strings" @@ -73,7 +72,7 @@ func TestNode_CreateNewNode(t *testing.T) { return } - if !compareNodes(got, tt.expectCurr) { + if !deepEqual(got, tt.expectCurr) { t.Errorf("%s got = %v, want %v", tt.name, got, tt.expectCurr) } }) @@ -1519,49 +1518,3 @@ func isSameObject(a, b string) bool { return true } - -func compareNodes(n1, n2 *Node) bool { - if n1 == nil || n2 == nil { - return n1 == n2 - } - - if n1.key != n2.key { - return false - } - - if !bytes.Equal(n1.data, n2.data) { - return false - } - - if n1.index != n2.index { - return false - } - - if n1.borders != n2.borders { - return false - } - - if n1.modified != n2.modified { - return false - } - - if n1.nodeType != n2.nodeType { - return false - } - - if !compareNodes(n1.prev, n2.prev) { - return false - } - - if len(n1.next) != len(n2.next) { - return false - } - - for k, v := range n1.next { - if !compareNodes(v, n2.next[k]) { - return false - } - } - - return true -} diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 15900476c52..699d05090b3 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -1,16 +1,56 @@ package json -// ref: https://support.smartbear.com/alertsite/docs/monitors/api/endpoint/jsonpath.html - import ( "errors" "io" + "math" + "strconv" "strings" - "unicode" + + "gno.land/p/demo/ufmt" ) -// ParsePath takes a JSONPath string and returns a slice of strings representing the path segments. -func ParsePath(path string) ([]string, error) { +const ( + errUnexpectedEOF = errors.New("unexpected EOF") + errUnexpectedChar = errors.New("unexpected character") + errStringNotClosed = errors.New("string not closed") + errBracketNotClosed = errors.New("bracket not closed") + errInvalidSlicePathSyntax = errors.New("invalid slice path syntax") + errInvalidSliceFromValue = errors.New("invalid slice from value") + errInvalidSliceToValue = errors.New("invalid slice to value") + errInvalidSliceStepValue = errors.New("invalid slice step value") +) + +// Path returns the nodes that match the given JSON path. +func Path(data []byte, path string) ([]*Node, error) { + commands, err := parsePath(path) + if err != nil { + return nil, ufmt.Errorf("failed to parse path: %v", err) + } + + nodes, err := Unmarshal(data) + if err != nil { + return nil, ufmt.Errorf("failed to unmarshal JSON: %v", err) + } + + return applyPath(nodes, commands) +} + +// parsePath parses the given path string and returns a slice of commands to be run. +// +// The function uses a state machine approach to parse the path based on the encountered tokens. +// It supports the following tokens and their corresponding states: +// - Dollar sign ('$'): Appends a literal "$" to the result slice. +// - Dot ('.'): Calls the processDot function to handle the dot token. +// - Single dot ('.') followed by a child end character: Appends the substring between the dots to the result slice. +// - Double dot ('..'): Appends ".." to the result slice. +// - Opening bracket ('['): Calls the processBracketOpen function to handle the opening bracket token. +// - Single quote ('”') after the opening bracket: Calls the processSingleQuote function to handle the string within single quotes. +// - Any other character after the opening bracket: Calls the processWithoutSingleQuote function to handle the string without single quotes. +// +// The function returns the slice of parsed commands and any error encountered during the parsing process. +// If an unexpected character is encountered, an error (errUnexpectedChar) is returned. +func parsePath(path string) ([]string, error) { buf := newBuffer([]byte(path)) result := make([]string, 0) @@ -19,365 +59,470 @@ func ParsePath(path string) ([]string, error) { if err != nil { break } + switch b { + case dollarSign: + result = append(result, "$") + case atSign: + result = append(result, "@") + case dot: + result, err = processDot(buf, result) + if err != nil { + return nil, err + } + case bracketOpen: + result, err = processBracketOpen(buf, result) + if err != nil { + return nil, err + } + default: + return nil, errUnexpectedChar + } - switch { - case b == dollarSign || b == atSign: - result = append(result, string(b)) - buf.step() + err = buf.step() + if err != nil && err != io.EOF { + return nil, err + } + } - case b == dot: - buf.step() + return result, nil +} - if next, _ := buf.current(); next == dot { - buf.step() - result = append(result, "..") +func applyPath(node *Node, cmds []string) ([]*Node, error) { + result := make([]*Node, 0) - extractNextSegment(buf, &result) - } else { - extractNextSegment(buf, &result) - } + for i, cmd := range cmds { + if i == 0 && (cmd == "$" || cmd == "@") { // root or current + result = append(result, node) + continue + } - case b == bracketOpen: - start := buf.index - buf.step() + var err error + result, err = processCommand(cmd, result) + if err != nil { + return nil, err + } + } - for { - if buf.index >= buf.length || buf.data[buf.index] == bracketClose { - break - } + return result, nil +} - buf.step() - } +// processCommand processes a single command on the given nodes. +// +// It determines the type of command and calls the corresponding function to handle the command. +func processCommand(cmd string, nodes []*Node) ([]*Node, error) { + switch { + case cmd == "..": + return processRecursiveDescent(nodes), nil + case cmd == "*": + return processWildcard(nodes), nil + case strings.Contains(cmd, ":"): + return processSlice(cmd, nodes) + case strings.HasPrefix(cmd, "?(") && strings.HasSuffix(cmd, ")"): + panic("filter not implemented") + default: + res, err := processKeyUnion(cmd, nodes) + if err != nil { + return nil, err + } - if buf.index >= buf.length { - return nil, errors.New("unexpected end of path") - } + return res, nil + } +} - segment := string(buf.sliceFromIndices(start+1, buf.index)) - result = append(result, segment) +// processWildcard processes a wildcard command on the given nodes. +// +// It retrieves all the child nodes of each node in the given slice. +func processWildcard(nodes []*Node) (result []*Node) { + for _, node := range nodes { + result = append(result, node.getSortedChildren()...) + } + return result +} - buf.step() +// processRecursiveDescent performs a recursive descent on the given nodes. +// +// It recursively retrieves all the child nodes of each node in the given slice. +func processRecursiveDescent(nodes []*Node) (result []*Node) { + for _, node := range nodes { + result = append(result, recursiveChildren(node)...) + } + return result +} - default: - buf.step() +// recursiveChildren returns all the recursive child nodes of the given node that are containers. +// +// It recursively traverses the child nodes of the given node and their child nodes, +// and returns a slice of pointers to all the child nodes that are containers. +func recursiveChildren(node *Node) (result []*Node) { + if node.isContainer() { + for _, element := range node.getSortedChildren() { + if element.isContainer() { + result = append(result, element) + } } } - return result, nil + temp := make([]*Node, 0, len(result)) + temp = append(temp, result...) + + for _, element := range result { + temp = append(temp, recursiveChildren(element)...) + } + + return temp } -// extractNextSegment extracts the segment from the current index -// to the next significant character and adds it to the resulting slice. -func extractNextSegment(buf *buffer, result *[]string) { +var pathSegmentDelimiters = map[byte]bool{dot: true, bracketOpen: true} + +// processDot handles the processing when a dot character is found in the buffer. +// +// It checks the next character in the buffer. +// If the next character is also a dot, it appends ".." to the result slice. +// Otherwise, it reads the characters until the next child end character (childEnd) and appends the substring to the result slice. +// It returns the updated result slice and any error encountered. +func processDot(buf *buffer, result []string) ([]string, error) { start := buf.index - buf.skipToNextSignificantToken() - if buf.index <= start { - return + b, err := buf.next() + if err == io.EOF { + err = nil + return result, nil } - segment := string(buf.sliceFromIndices(start, buf.index)) - if segment != "" { - *result = append(*result, segment) + if err != nil { + return nil, err + } + + if b == dot { + result = append(result, "..") + buf.index-- + return result, nil + } + + err = buf.skipAny(pathSegmentDelimiters) + stop := buf.index + + if err == io.EOF { + err = nil + stop = buf.length + } else { + buf.index-- + } + + if err != nil { + return nil, err + } + + if start+1 < stop { + result = append(result, string(buf.data[start+1:stop])) } -} -// numIndex holds a map of valid numeric characters -var numIndex = map[byte]bool{ - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - '.': true, - 'e': true, - 'E': true, + return result, nil } -// pathToken checks if the current token is a valid JSON path token. -func (b *buffer) pathToken() error { - var ( - stack []byte - inToken, inNumber bool - ) +// processBracketOpen handles the processing when an opening bracket character ('[') is found in the buffer. +// +// It reads the next character in the buffer and determines the appropriate processing based on the character: +// - If the next character is a single quote (`'`), it calls the processSingleQuote function to handle the string within single quotes. +// - Otherwise, it calls the processWithoutSingleQuote function to handle the string without single quotes. +// +// It returns the updated result slice and any error encountered. +func processBracketOpen(buf *buffer, result []string) ([]string, error) { + b, err := buf.next() + if err != nil { + return nil, errUnexpectedEOF + } - first := b.index + start := buf.index + if b == singleQuote { + result, err = processSingleQuote(buf, result, start) + } else { + result, err = processWithoutSingleQuote(buf, result, start) + } - for b.index < b.length { - c := b.data[b.index] + if err != nil { + return nil, err + } - switch { - case c == doubleQuote, c == singleQuote: - inToken = true - if err := b.step(); err != nil { - return errors.New("error stepping through buffer") - } + return result, nil +} - if err := b.skip(c); err != nil { - return errors.New("unmatched quote in path") - } +// processSingleQuote handles the processing when a single quote character (`'`) is encountered after an opening bracket ('[') in the buffer. +// +// It assumes that the current position of the buffer is just after the single quote character. +// +// The function performs the following steps: +// 1. It skips the single quote character and reads the string until the next single quote character is found. +// 2. It checks if the character after the closing single quote is a closing bracket (']'). +// - If it is, the string between the single quotes is appended to the result slice. +// - If it is not, an error (errBracketNotClosed) is returned. +// +// It returns the updated result slice and any error encountered. +func processSingleQuote(buf *buffer, result []string, start int) ([]string, error) { + start++ + + err := buf.string(singleQuote, true) + if err != nil { + return nil, errStringNotClosed + } - if b.index >= b.length { - return errors.New("unmatched quote in path") - } + stop := buf.index - case c == bracketOpen, c == parenOpen: - inToken = true - stack = append(stack, c) + b, err := buf.next() + if err != nil { + return nil, errUnexpectedEOF + } - case c == bracketClose, c == parenClose: - inToken = true - ls := len(stack) - if ls == 0 { - return errors.New("unexpected end of path") - } + if b != bracketClose { + return nil, errBracketNotClosed + } - if (c == bracketClose && stack[ls-1] != bracketOpen) || (c == parenClose && stack[ls-1] != parenOpen) { - return errors.New("mismatched bracket or parenthesis") - } + result = append(result, string(buf.data[start:stop])) - stack = stack[:ls-1] + return result, nil +} - case pathStateContainsValidPathToken(c): - inToken = true +// processWithoutSingleQuote handles the processing when a character other than +// a single quote (`'`) is encountered after an opening bracket ('[') in the buffer. +// +// It assumes that the current position of the buffer is just after the opening bracket. +// +// The function reads the characters until the next closing bracket (']') is found +// and appends the substring between the brackets to the result slice. +// +// It returns the updated result slice and any error encountered. +// If the closing bracket is not found, an error (errUnexpectedEOF) is returned. +func processWithoutSingleQuote(buf *buffer, result []string, start int) ([]string, error) { + err := buf.skip(bracketClose) + if err != nil { + return nil, errUnexpectedEOF + } - case c == plus, c == minus: - if inNumber || (b.index > 0 && numIndex[b.data[b.index-1]]) { - inToken = true - } + stop := buf.index + result = append(result, string(buf.data[start:stop])) - if !inToken && (b.index+1 < b.length && numIndex[b.data[b.index+1]]) { - inToken = true - inNumber = true - } + return result, nil +} - if !inToken { - return errors.New("unexpected operator at start of token") - } +// processSlice processes a slice path on the given nodes. +// +// The slice path has the following syntax: +// +// [start:end:step] +// +// - start: The starting index of the slice (inclusive). if omitted, it defaults to 0. +// If negative, it counts from the end of the array. +// - end: The ending index of the slice (exclusive). if omitted, it defaults to the length of the array. +// If negative, it counts from the end of the array. +// - step: The step value for the slice. if omitted, it defaults to 1. +// +// The function performs the following steps: +// +// 1. Split the slice path into start, end, and step values. +// +// 2. Parses the each syntax components as integers. +// +// 3. For each node in the given nodes: +// - If the node is an array: +// - Calculate the length of the array. +// - Adjust the start and end values if they are negative. +// - Check if the slice range is within the bounds of the array. +// - Iterate over the array elements based on the start, end and step values. +// - Append the selected elements to the result slice. +// +// It returns the slice of selected nodes and any error encountered during the parsing process. +func processSlice(cmd string, nodes []*Node) ([]*Node, error) { + from, to, step, err := parseSliceParams(cmd) + if err != nil { + return nil, err + } - default: - if len(stack) != 0 || inToken { - inToken = true - } else { - goto end - } + var result []*Node + for _, node := range nodes { + if node.IsArray() { + result = append(result, selectArrayElement(node, from, to, step)...) } + } + + return result, nil +} - b.index++ +// parseSliceParams parses the slice parameters from the given path command. +func parseSliceParams(cmd string) (int64, int64, int64, error) { + keys := strings.Split(cmd, ":") + ks := len(keys) + if ks > 3 { + return 0, 0, 0, errInvalidSlicePathSyntax } -end: - if len(stack) != 0 { - return errors.New("unclosed bracket or parenthesis at end of path") + from, err := strconv.ParseInt(keys[0], 10, 64) + if err != nil { + return 0, 0, 0, errInvalidSliceFromValue } - if first == b.index { - return errors.New("no token found") + to := int64(0) + if ks > 1 { + to, err = strconv.ParseInt(keys[1], 10, 64) + if err != nil { + return 0, 0, 0, errInvalidSliceToValue + } } - if inNumber && !numIndex[b.data[b.index-1]] { - inNumber = false + step := int64(1) + if ks == 3 { + step, err = strconv.ParseInt(keys[2], 10, 64) + if err != nil { + return 0, 0, 0, errInvalidSliceStepValue + } } - return nil + return from, to, step, nil } -func pathStateContainsValidPathToken(c byte) bool { - if _, ok := significantTokens[c]; ok { - return true +// selectArrayElement selects the array elements based on the given from, to, and step values. +func selectArrayElement(node *Node, from, to, step int64) []*Node { + length := int64(len(node.next)) + + if to == 0 { + to = length } - if _, ok := filterTokens[c]; ok { - return true + if from < 0 { + from += length } - if _, ok := numIndex[c]; ok { - return true + if to < 0 { + to += length } - if unicode.IsLetter(rune(c)) { - return true + from = int64(math.Max(0, math.Min(float64(from), float64(length)))) + to = int64(math.Max(0, math.Min(float64(to), float64(length)))) + + if step <= 0 || from >= to { + return nil + } + + // This formula calculates the number of elements that will be selected based on the given + // from, to, and step values. It ensures that the correct number of elements are allocated + // in the result slice. + size := (to - from + step - 1) / step + result := make([]*Node, 0, size) + + for i := from; i < to; i += step { + if child, ok := node.next[ufmt.Sprintf("%d", i)]; ok { + result = append(result, child) + } } - return false + return result } -func (b *buffer) pathTokenizer() ([]string, error) { - var result []string +// processKeyUnion processes a key union command on the given nodes. +// +// It retrieves the child nodes of each node in the given slice that match any of the specified keys +func processKeyUnion(cmd string, nodes []*Node) ([]*Node, error) { + buf := newBuffer([]byte(cmd)) + keys, err := extractKeys(buf) + if err != nil { + return nil, err + } - for { - b.reset() - c, err := b.first() - if err == io.EOF { - break + var result []*Node + for _, node := range nodes { + if node.IsArray() { + result, err = processArrayKeys(node, keys, result) + } else if node.IsObject() { + result, err = processObjectKeys(node, keys, result) } + if err != nil { return nil, err } + } + return result, nil +} - var token string - switch { - case unicode.IsDigit(rune(c)), c == dot: - start := b.index - err = b.numeric(true) - if err == io.EOF { - token = string(b.sliceFromIndices(start, b.index)) - } else if err != nil { - if c == dot { - token = "." - b.index = start - } else { - return nil, err - } - } else { - token = string(b.sliceFromIndices(start, b.index)) - b.index-- - } +func extractKeys(buf *buffer) ([]string, error) { + keys := make([]string, 0) - case c == singleQuote, c == doubleQuote: - start := b.index - err = b.string(c, true) - if err != nil { - return nil, errors.New("error stepping through buffer") - } - token = string(b.sliceFromIndices(start, b.index+1)) + for { + key, err := extractKey(buf) + if err != nil && err != io.EOF { + return nil, err + } - case c == dollarSign, c == atSign: - start := b.index - err = b.findNextToken() - if err != nil && err != io.EOF { - return nil, err - } - token = string(b.sliceFromIndices(start, b.index)) - if err != io.EOF { - b.index-- - } + keys = append(keys, key) - case c == parenOpen, c == parenClose: - token = string(c) + if err := expectComma(buf); err != nil { + return keys, nil + } + } - default: - start := b.index - for ; b.index < b.length; b.index++ { - c = b.data[b.index] - if c == parenOpen { - break - } - - rc := rune(c) - if unicode.IsLetter(rc) && !unicode.IsDigit(rc) && c != '_' { - break - } - } - err = b.step() - if err != nil && err != io.EOF { - return nil, err - } + return keys, nil +} - slice := string(b.sliceFromIndices(start, b.index)) - token = strings.ToLower(slice) +func extractKey(buf *buffer) (string, error) { + from := buf.index + err := buf.pathToken() + if err != nil { + return "", err + } - if err != io.EOF { - b.index-- - } - } + key := string(buf.data[from:buf.index]) + if len(key) > 2 && key[0] == singleQuote && key[len(key)-1] == singleQuote { + key = key[1 : len(key)-1] + } + return key, nil +} - result = append(result, token) - err = b.step() - if err == io.EOF { - break - } +func expectComma(buf *buffer) error { + c, err := buf.first() + if err != nil { + return err + } + + if c != comma { + return errUnexpectedChar + } + + return buf.step() +} + +func processArrayKeys(node *Node, keys []string, result []*Node) ([]*Node, error) { + for _, key := range keys { + index, err := strconv.Atoi(key) if err != nil { return nil, err } + + if index < 0 { + index = node.Size() + index + } + + if value, ok := node.next[ufmt.Sprintf("%d", index)]; ok { + result = append(result, value) + } } return result, nil } -func (b *buffer) findNextToken() error { - var ( - c byte - stack []byte - find bool - start int - ) - - for b.index < b.length { - c = b.data[b.index] - - switch { - case c == singleQuote, c == doubleQuote: - find = true - if err := b.step(); err != nil { - return errors.New("error stepping through buffer") - } - if err := b.skip(c); err == io.EOF { - return errors.New("unmatched quote in path") - } - case c == bracketOpen, c == parenOpen: - find = true - stack = append(stack, c) - case c == bracketClose, c == parenClose: - find = true - ls := len(stack) - if ls == 0 { - if start == b.index { - return errors.New("unexpected end of path") - } - break - } - expected := bracketOpen - if c == parenClose { - expected = parenOpen - } - if rune(stack[ls-1]) != expected { - return errors.New("mismatched bracket or parenthesis") - } - stack = stack[:ls-1] - case c == dot, c == atSign, c == dollarSign, c == question, c == asterisk, isAlphaNumeric(c): - find = true - b.index++ - continue - case len(stack) != 0: - find = true - b.index++ - continue - case c == minus, c == plus: - if !find { - find = true - start = b.index - if err := b.numeric(true); err == nil || err == io.EOF { - b.index-- - b.index++ - continue - } - b.index = start - } - fallthrough - default: - break +func processObjectKeys(node *Node, keys []string, result []*Node) ([]*Node, error) { + for _, key := range keys { + if value, ok := node.next[key]; ok { + result = append(result, value) } - - if len(stack) == 0 { - break - } - b.index++ } - if len(stack) != 0 { - return errors.New("unclosed bracket or parenthesis at end of path") - } - if start == b.index { - return b.step() - } - if b.index >= b.length { - return io.EOF + return result, nil +} + +// Paths returns calculated paths of underlying nodes +func Paths(array []*Node) []string { + result := make([]string, 0, len(array)) + for _, element := range array { + result = append(result, element.Path()) } - return nil + + return result } diff --git a/examples/gno.land/p/demo/json/path_test.gno b/examples/gno.land/p/demo/json/path_test.gno index eb175c4ee82..490aeb7fd90 100644 --- a/examples/gno.land/p/demo/json/path_test.gno +++ b/examples/gno.land/p/demo/json/path_test.gno @@ -1,123 +1,248 @@ package json import ( + "strings" "testing" ) -func TestPath_ParseJSONPath(t *testing.T) { +func TestParseJSONPath(t *testing.T) { tests := []struct { - name string path string expected []string }{ - {name: "Empty string path", path: "", expected: []string{}}, - {name: "Root only path", path: "$", expected: []string{"$"}}, - {name: "Root with dot path", path: "$.", expected: []string{"$"}}, - {name: "All objects in path", path: "$..", expected: []string{"$", ".."}}, - {name: "Only children in path", path: "$.*", expected: []string{"$", "*"}}, - {name: "All objects' children in path", path: "$..*", expected: []string{"$", "..", "*"}}, - {name: "Simple dot notation path", path: "$.root.element", expected: []string{"$", "root", "element"}}, - {name: "Complex dot notation path with wildcard", path: "$.root.*.element", expected: []string{"$", "root", "*", "element"}}, - {name: "Path with array wildcard", path: "$.phoneNumbers[*].type", expected: []string{"$", "phoneNumbers", "*", "type"}}, - {name: "Path with filter expression", path: "$.store.book[?(@.price < 10)].title", expected: []string{"$", "store", "book", "?(@.price < 10)", "title"}}, - {name: "Path with formula", path: "$..phoneNumbers..('ty' + 'pe')", expected: []string{"$", "..", "phoneNumbers", "..", "('ty' + 'pe')"}}, - {name: "Simple bracket notation path", path: "$['root']['element']", expected: []string{"$", "'root'", "'element'"}}, - {name: "Complex bracket notation path with wildcard", path: "$['root'][*]['element']", expected: []string{"$", "'root'", "*", "'element'"}}, - {name: "Bracket notation path with integer index", path: "$['store']['book'][0]['title']", expected: []string{"$", "'store'", "'book'", "0", "'title'"}}, - {name: "Complex path with wildcard in bracket notation", path: "$['root'].*['element']", expected: []string{"$", "'root'", "*", "'element'"}}, - {name: "Mixed notation path with dot after bracket", path: "$.['root'].*.['element']", expected: []string{"$", "'root'", "*", "'element'"}}, - {name: "Mixed notation path with dot before bracket", path: "$['root'].*.['element']", expected: []string{"$", "'root'", "*", "'element'"}}, - {name: "Single character path with root", path: "$.a", expected: []string{"$", "a"}}, - {name: "Multiple characters path with root", path: "$.abc", expected: []string{"$", "abc"}}, - {name: "Multiple segments path with root", path: "$.a.b.c", expected: []string{"$", "a", "b", "c"}}, - {name: "Multiple segments path with wildcard and root", path: "$.a.*.c", expected: []string{"$", "a", "*", "c"}}, - {name: "Multiple segments path with filter and root", path: "$.a[?(@.b == 'c')].d", expected: []string{"$", "a", "?(@.b == 'c')", "d"}}, - {name: "Complex path with multiple filters", path: "$.a[?(@.b == 'c')].d[?(@.e == 'f')].g", expected: []string{"$", "a", "?(@.b == 'c')", "d", "?(@.e == 'f')", "g"}}, - {name: "Complex path with multiple filters and wildcards", path: "$.a[?(@.b == 'c')].*.d[?(@.e == 'f')].g", expected: []string{"$", "a", "?(@.b == 'c')", "*", "d", "?(@.e == 'f')", "g"}}, - {name: "Path with array index and root", path: "$.a[0].b", expected: []string{"$", "a", "0", "b"}}, - {name: "Path with multiple array indices and root", path: "$.a[0].b[1].c", expected: []string{"$", "a", "0", "b", "1", "c"}}, - {name: "Path with array index, wildcard and root", path: "$.a[0].*.c", expected: []string{"$", "a", "0", "*", "c"}}, + {path: "$", expected: []string{"$"}}, + {path: "$.", expected: []string{"$"}}, + {path: "$..", expected: []string{"$", ".."}}, + {path: "$.*", expected: []string{"$", "*"}}, + {path: "$..*", expected: []string{"$", "..", "*"}}, + {path: "$.root.element", expected: []string{"$", "root", "element"}}, + {path: "$.root.*.element", expected: []string{"$", "root", "*", "element"}}, + {path: "$['root']['element']", expected: []string{"$", "root", "element"}}, + {path: "$['root'][*]['element']", expected: []string{"$", "root", "*", "element"}}, + {path: "$['store']['book'][0]['title']", expected: []string{"$", "store", "book", "0", "title"}}, + {path: "$['root'].*['element']", expected: []string{"$", "root", "*", "element"}}, + {path: "$.['root'].*.['element']", expected: []string{"$", "root", "*", "element"}}, + {path: "$['root'].*.['element']", expected: []string{"$", "root", "*", "element"}}, + {path: "$.phoneNumbers[*].type", expected: []string{"$", "phoneNumbers", "*", "type"}}, + // TODO: support filter expressions + // {path: "$.store.book[?(@.price < 10)].title", expected: []string{"$", "store", "book", "?(@.price < 10)", "title"}}, + } + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + result, err := parsePath(tt.path) + if err != nil { + t.Errorf("error on path %s: %s", tt.path, err.Error()) + } else if !sliceEqual(result, tt.expected) { + t.Errorf("expected %s, got %s", sliceString(tt.expected), sliceString(result)) + } + }) + } +} + +func TestArrayPath(t *testing.T) { + data := `{ + "numbers": [1, 2, 3, 4, 5], + "colors": ["red", "green", "blue"], + "nested": [ + { + "name": "Alice", + "age": 30 + }, + { + "name": "Bob", + "age": 25 + }, + { + "name": "Charlie", + "age": 35 + } + ] + }` + + tests := []struct { + name string + path string + expected string + isErr bool + }{ + {name: "array index", path: "$.numbers[2]", expected: "[$['numbers'][2]]"}, + {name: "array slice", path: "$.numbers[1:4]", expected: "[$['numbers'][1], $['numbers'][2], $['numbers'][3]]"}, + {name: "array slice with step", path: "$.numbers[0:5:2]", expected: "[$['numbers'][0], $['numbers'][2], $['numbers'][4]]"}, + {name: "array slice with negative index and step", path: "$.colors[-3:-1:1]", expected: "[$['colors'][0], $['colors'][1]]"}, + {name: "nested array slice", path: "$.nested[0:2].name", expected: "[$['nested'][0]['name'], $['nested'][1]['name']]"}, + {name: "nested array slice with step", path: "$.nested[0:3:2].age", expected: "[$['nested'][0]['age'], $['nested'][2]['age']]"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - reult, _ := ParsePath(tt.path) - if !isEqualSlice(reult, tt.expected) { - t.Errorf("ParsePath(%s) expected: %v, got: %v", tt.path, tt.expected, reult) + res, err := Path([]byte(data), tt.path) + if (err != nil) != tt.isErr { + t.Errorf("expected error %v, got %v", tt.isErr, err) + } + if tt.isErr { + return + } + if fullPath(res) != tt.expected { + t.Errorf("expected %s, got %s", tt.expected, fullPath(res)) } }) } } -func TestPath_pathTokenizer(t *testing.T) { +func TestApplyPath(t *testing.T) { + node1 := NumberNode("", 1) + node2 := NumberNode("", 2) + cpy := func(n Node) *Node { + return &n + } + array := ArrayNode("", []*Node{cpy(*node1), cpy(*node2)}) + + type pathData struct { + node *Node + path []string + } + tests := []struct { - name string - want []string - isErr bool + name string + args pathData + expected []*Node + isErr bool }{ { - name: "", - want: []string{}, + name: "root", + args: pathData{ + node: node1, + path: []string{"$"}, + }, + expected: []*Node{node1}, }, { - name: "1234", - want: []string{"1234"}, + name: "second", + args: pathData{ + node: array, + path: []string{"$", "1"}, + }, + expected: []*Node{array.next["1"]}, }, { - name: "12e34", - want: []string{"12e34"}, + name: "object", + args: pathData{ + node: ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + path: []string{"$", "key1"}, + }, + expected: []*Node{NumberNode("", 42)}, }, { - name: "12.34", - want: []string{"12.34"}, + name: "wildcard on object", + args: pathData{ + node: ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + path: []string{"$", "*"}, + }, + expected: []*Node{NumberNode("", 42), NumberNode("", 43)}, }, { - name: "'foo'", - want: []string{"'foo'"}, + name: "array slice", + args: pathData{ + node: ArrayNode("", []*Node{ + NumberNode("", 1), + NumberNode("", 2), + NumberNode("", 3), + NumberNode("", 4), + }), + path: []string{"$", "1:3"}, + }, + expected: []*Node{NumberNode("", 2), NumberNode("", 3)}, }, { - name: "@.length", - want: []string{"@.length"}, + name: "array slice with step", + args: pathData{ + node: ArrayNode("", []*Node{ + NumberNode("", 1), + NumberNode("", 2), + NumberNode("", 3), + NumberNode("", 4), + NumberNode("", 5), + }), + path: []string{"$", "0:5:2"}, + }, + expected: []*Node{ + NumberNode("", 1), + NumberNode("", 3), + NumberNode("", 5), + }, }, { - name: "$.var2", - want: []string{"$.var2"}, + name: "array slice with negative index and step", + args: pathData{ + node: ArrayNode("", []*Node{ + NumberNode("", 1), + NumberNode("", 2), + NumberNode("", 3), + NumberNode("", 4), + NumberNode("", 5), + }), + path: []string{"$", "-3:-1:1"}, + }, + expected: []*Node{ + NumberNode("", 3), + NumberNode("", 4), + }, }, { - name: "(@.length)", - want: []string{"(", "@.length", ")"}, + name: "basic recursive descent path", + args: pathData{ + node: ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + path: []string{"$", ".."}, + }, + expected: []*Node{ + ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + NumberNode("", 42), + NumberNode("", 43), + }, }, { - name: `"{"`, - want: []string{`"{"`}, - }, - { - name: "@.[", - isErr: true, + name: "recursive descent with wildcard", + args: pathData{ + node: ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + path: []string{"$", "..", "*"}, + }, + expected: []*Node{ + ObjectNode("", map[string]*Node{ + "key1": NumberNode("", 42), + "key2": NumberNode("", 43), + }), + NumberNode("", 42), + NumberNode("", 41), + }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := &buffer{ - data: []byte(tt.name), - length: len(tt.name), - } - - got, err := b.pathTokenizer() + result, err := applyPath(tt.args.node, tt.args.path) if (err != nil) != tt.isErr { - t.Errorf("pathTokenizer() error = %v, wantErr %v", err, tt.isErr) - return - } - - if !isEqualSlice(got, tt.want) { - t.Errorf("pathTokenizer() got = %v, want %v", got, tt.want) + t.Errorf("#1 expected error %v, got %v", tt.isErr, err) } }) } } -func isEqualSlice(a, b []string) bool { +func sliceEqual(a, b []string) bool { if len(a) != len(b) { return false } @@ -130,3 +255,16 @@ func isEqualSlice(a, b []string) bool { return true } + +func fullPath(array []*Node) string { + return sliceString(Paths(array)) +} + +func sliceString(array []string) string { + return "[" + strings.Join(array, ", ") + "]" +} + +func (n *Node) Equals(other *Node) bool { + // Compare the values of n and other + return n.value == other.value && n.nodeType == other.nodeType +} From bb071cced7ddd17bd693c1a162964720eb38b0d3 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 12 Jun 2024 15:26:27 +0900 Subject: [PATCH 5/7] fix: array expression does not recognize the ".." properly --- examples/gno.land/p/demo/json/buffer.gno | 1 - examples/gno.land/p/demo/json/node.gno | 30 ++--- examples/gno.land/p/demo/json/path.gno | 29 +++-- gnovm/tests/files/json0.gno | 151 +++++++++++++++++++++++ 4 files changed, 177 insertions(+), 34 deletions(-) create mode 100644 gnovm/tests/files/json0.gno diff --git a/examples/gno.land/p/demo/json/buffer.gno b/examples/gno.land/p/demo/json/buffer.gno index fd459ed69ed..d726ffadc7d 100644 --- a/examples/gno.land/p/demo/json/buffer.gno +++ b/examples/gno.land/p/demo/json/buffer.gno @@ -3,7 +3,6 @@ package json import ( "errors" "io" - "strings" "gno.land/p/demo/ufmt" ) diff --git a/examples/gno.land/p/demo/json/node.gno b/examples/gno.land/p/demo/json/node.gno index cd4cd45cc0f..e6f8648ba10 100644 --- a/examples/gno.land/p/demo/json/node.gno +++ b/examples/gno.land/p/demo/json/node.gno @@ -1,9 +1,7 @@ package json import ( - "bytes" "errors" - "sort" "strconv" "strings" @@ -1130,20 +1128,6 @@ func (n *Node) Keys() []string { return result } -type nodeKeys []string - -func (keys nodeKeys) Len() int { - return len(keys) -} - -func (keys nodeKeys) Less(i, j int) bool { - return keys[i] < keys[j] -} - -func (keys nodeKeys) Swap(i, j int) { - keys[i], keys[j] = keys[j], keys[i] -} - func (n *Node) getSortedChildren() (result []*Node) { if n == nil { return nil @@ -1152,8 +1136,18 @@ func (n *Node) getSortedChildren() (result []*Node) { size := len(n.next) if n.IsObject() { result = make([]*Node, size) - keys := nodeKeys(n.Keys()) - sort.Sort(keys) + keys := n.Keys() + + // sort keys in ascending order + for i := 1; i < len(keys); i++ { + key := keys[i] + j := i - 1 + for j >= 0 && keys[j] > key { + keys[j+1] = keys[j] + j-- + } + keys[j+1] = key + } for i, key := range keys { result[i] = n.next[key] diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 699d05090b3..4ef0f5621ea 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -10,7 +10,7 @@ import ( "gno.land/p/demo/ufmt" ) -const ( +var ( errUnexpectedEOF = errors.New("unexpected EOF") errUnexpectedChar = errors.New("unexpected character") errStringNotClosed = errors.New("string not closed") @@ -463,8 +463,7 @@ func extractKeys(buf *buffer) ([]string, error) { func extractKey(buf *buffer) (string, error) { from := buf.index - err := buf.pathToken() - if err != nil { + if err := buf.pathToken(); err != nil { return "", err } @@ -490,20 +489,20 @@ func expectComma(buf *buffer) error { func processArrayKeys(node *Node, keys []string, result []*Node) ([]*Node, error) { for _, key := range keys { - index, err := strconv.Atoi(key) - if err != nil { - return nil, err - } - - if index < 0 { - index = node.Size() + index - } - - if value, ok := node.next[ufmt.Sprintf("%d", index)]; ok { - result = append(result, value) + switch key { + default: + index, err := strconv.Atoi(key) + if err == nil { + if index < 0 { + index = node.Size() + index + } + + if value, ok := node.next[strconv.Itoa(index)]; ok { + result = append(result, value) + } + } } } - return result, nil } diff --git a/gnovm/tests/files/json0.gno b/gnovm/tests/files/json0.gno new file mode 100644 index 00000000000..b77ef0e0e63 --- /dev/null +++ b/gnovm/tests/files/json0.gno @@ -0,0 +1,151 @@ +package main + +import ( + "gno.land/p/demo/json" + "gno.land/p/demo/ufmt" +) + +func main() { + data := []byte(`{ + "store": { + "book": [ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J.R.R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + }, + "expensive": 10 + }`) + + paths := []string{ + "$.store.*", // All direct properties of `store` (not recursive) + "$.store.bicycle.color", // The color of the bicycle in the store (result: red) + "$.store.book[*]", // All books in the store + "$.store.book[0].title", // The title of the first book + + "$.store..price", // The prices of all items in the store + "$..price", // Result: [8.95, 8.99, 22.99, 19.95] + "$..book[*].title", // The titles of all books in the store + "$..book[0]", // The first book + } + + for _, path := range paths { + result, err := json.Path(data, path) + if err != nil { + ufmt.Println(err) + return + } + + ufmt.Println("Path:", path) + + for _, node := range result { + ufmt.Println(node.String()) + } + + ufmt.Println() + } +} + +// output: +// +// Path: $.store.* +// { +// "color": "red", +// "price": 19.95 +// } +// [ +// { +// "category": "reference", +// "author": "Nigel Rees", +// "title": "Sayings of the Century", +// "price": 8.95 +// }, +// { +// "category": "fiction", +// "author": "Herman Melville", +// "title": "Moby Dick", +// "isbn": "0-553-21311-3", +// "price": 8.99 +// }, +// { +// "category": "fiction", +// "author": "J.R.R. Tolkien", +// "title": "The Lord of the Rings", +// "isbn": "0-395-19395-8", +// "price": 22.99 +// } +// ] +// +// Path: $.store.bicycle.color +// "red" +// +// Path: $.store.book[*] +// { +// "category": "reference", +// "author": "Nigel Rees", +// "title": "Sayings of the Century", +// "price": 8.95 +// } +// { +// "category": "fiction", +// "author": "Herman Melville", +// "title": "Moby Dick", +// "isbn": "0-553-21311-3", +// "price": 8.99 +// } +// { +// "category": "fiction", +// "author": "J.R.R. Tolkien", +// "title": "The Lord of the Rings", +// "isbn": "0-395-19395-8", +// "price": 22.99 +// } +// +// Path: $.store.book[0].title +// "Sayings of the Century" +// +// Path: $.store..price +// 19.95 +// 8.95 +// 8.99 +// 22.99 +// +// Path: $..price +// 19.95 +// 8.95 +// 8.99 +// 22.99 +// +// Path: $..book[*].title +// "Sayings of the Century" +// "Moby Dick" +// "The Lord of the Rings" +// +// Path: $..book[0] +// { +// "category": "reference", +// "author": "Nigel Rees", +// "title": "Sayings of the Century", +// "price": 8.95 +// } From 0d9b265813201afab3d79395fff2535c424da0d8 Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 12 Jun 2024 15:57:43 +0900 Subject: [PATCH 6/7] Apply caching to avoid unnecessary unmarshals --- examples/gno.land/p/demo/json/path.gno | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 4ef0f5621ea..5ab343e949f 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -1,6 +1,8 @@ package json import ( + "crypto/sha256" + "encoding/hex" "errors" "io" "math" @@ -21,6 +23,15 @@ var ( errInvalidSliceStepValue = errors.New("invalid slice step value") ) +// caching nodes to avoid unmarshalling the same JSON data multiple times +var cacheNode = make(map[string]*Node) + +// generateCacheKey creates a hash of the data to be used as a cache key. +func generateCacheKey(data []byte) string { + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + // Path returns the nodes that match the given JSON path. func Path(data []byte, path string) ([]*Node, error) { commands, err := parsePath(path) @@ -28,9 +39,17 @@ func Path(data []byte, path string) ([]*Node, error) { return nil, ufmt.Errorf("failed to parse path: %v", err) } - nodes, err := Unmarshal(data) - if err != nil { - return nil, ufmt.Errorf("failed to unmarshal JSON: %v", err) + dataKey := generateCacheKey(data) + nodes, ok := cacheNode[dataKey] + + println("dataKey", dataKey, ok) + + if !ok { + nodes, err = Unmarshal(data) + if err != nil { + return nil, ufmt.Errorf("failed to unmarshal JSON: %v", err) + } + cacheNode[dataKey] = nodes } return applyPath(nodes, commands) From 553a6ee9b98d442f0340f1bb55b6c67c28d646ff Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Wed, 12 Jun 2024 16:43:50 +0900 Subject: [PATCH 7/7] fix: test --- examples/gno.land/p/demo/json/path.gno | 2 - examples/gno.land/p/demo/json/path_test.gno | 318 ++++++++------------ gnovm/tests/files/json0.gno | 151 ---------- 3 files changed, 124 insertions(+), 347 deletions(-) delete mode 100644 gnovm/tests/files/json0.gno diff --git a/examples/gno.land/p/demo/json/path.gno b/examples/gno.land/p/demo/json/path.gno index 5ab343e949f..6048148c4f6 100644 --- a/examples/gno.land/p/demo/json/path.gno +++ b/examples/gno.land/p/demo/json/path.gno @@ -42,8 +42,6 @@ func Path(data []byte, path string) ([]*Node, error) { dataKey := generateCacheKey(data) nodes, ok := cacheNode[dataKey] - println("dataKey", dataKey, ok) - if !ok { nodes, err = Unmarshal(data) if err != nil { diff --git a/examples/gno.land/p/demo/json/path_test.gno b/examples/gno.land/p/demo/json/path_test.gno index 490aeb7fd90..2692c456d55 100644 --- a/examples/gno.land/p/demo/json/path_test.gno +++ b/examples/gno.land/p/demo/json/path_test.gno @@ -3,6 +3,7 @@ package json import ( "strings" "testing" + "unicode" ) func TestParseJSONPath(t *testing.T) { @@ -39,232 +40,161 @@ func TestParseJSONPath(t *testing.T) { } } -func TestArrayPath(t *testing.T) { - data := `{ - "numbers": [1, 2, 3, 4, 5], - "colors": ["red", "green", "blue"], - "nested": [ +func TestJsonPath(t *testing.T) { + // JSON from: https://support.smartbear.com/alertsite/docs/monitors/api/endpoint/jsonpath.html + data := []byte(`{ + "store": { + "book": [ { - "name": "Alice", - "age": 30 + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 }, { - "name": "Bob", - "age": 25 + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 }, { - "name": "Charlie", - "age": 35 + "category": "fiction", + "author": "J.R.R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 } - ] - }` + ], + "bicycle": { + "color": "red", + "price": 19.95 + } + }, + "expensive": 10 + }`) tests := []struct { - name string path string - expected string - isErr bool + expected []string }{ - {name: "array index", path: "$.numbers[2]", expected: "[$['numbers'][2]]"}, - {name: "array slice", path: "$.numbers[1:4]", expected: "[$['numbers'][1], $['numbers'][2], $['numbers'][3]]"}, - {name: "array slice with step", path: "$.numbers[0:5:2]", expected: "[$['numbers'][0], $['numbers'][2], $['numbers'][4]]"}, - {name: "array slice with negative index and step", path: "$.colors[-3:-1:1]", expected: "[$['colors'][0], $['colors'][1]]"}, - {name: "nested array slice", path: "$.nested[0:2].name", expected: "[$['nested'][0]['name'], $['nested'][1]['name']]"}, - {name: "nested array slice with step", path: "$.nested[0:3:2].age", expected: "[$['nested'][0]['age'], $['nested'][2]['age']]"}, + {"$.store.*", []string{ + `{ + "color": "red", + "price": 19.95 +}`, + `[ + { + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, + { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, + { + "category": "fiction", + "author": "J.R.R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + } +]`, + }}, + {"$.store.bicycle.color", []string{`"red"`}}, + {"$.store.book[*]", []string{ + `{ + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 +}`, + `{ + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 +}`, + `{ + "category": "fiction", + "author": "J.R.R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 +}`, + }}, + {"$.store.book[0].title", []string{`"Sayings of the Century"`}}, + {"$.store..price", []string{`19.95`, `8.95`, `8.99`, `22.99`}}, + {"$..price", []string{`19.95`, `8.95`, `8.99`, `22.99`}}, + {"$..book[*].title", []string{`"Sayings of the Century"`, `"Moby Dick"`, `"The Lord of the Rings"`}}, + {"$..book[0]", []string{ + `{ + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 +}`, + }}, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res, err := Path([]byte(data), tt.path) - if (err != nil) != tt.isErr { - t.Errorf("expected error %v, got %v", tt.isErr, err) - } - if tt.isErr { - return - } - if fullPath(res) != tt.expected { - t.Errorf("expected %s, got %s", tt.expected, fullPath(res)) - } - }) - } -} - -func TestApplyPath(t *testing.T) { - node1 := NumberNode("", 1) - node2 := NumberNode("", 2) - cpy := func(n Node) *Node { - return &n - } - array := ArrayNode("", []*Node{cpy(*node1), cpy(*node2)}) + result, err := Path(data, tt.path) + if err != nil { + t.Errorf("Unexpected error for path %q: %v", tt.path, err) + continue + } - type pathData struct { - node *Node - path []string - } + if len(result) != len(tt.expected) { + t.Errorf("Path %q: expected %d results, got %d", tt.path, len(tt.expected), len(result)) + continue + } - tests := []struct { - name string - args pathData - expected []*Node - isErr bool - }{ - { - name: "root", - args: pathData{ - node: node1, - path: []string{"$"}, - }, - expected: []*Node{node1}, - }, - { - name: "second", - args: pathData{ - node: array, - path: []string{"$", "1"}, - }, - expected: []*Node{array.next["1"]}, - }, - { - name: "object", - args: pathData{ - node: ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - path: []string{"$", "key1"}, - }, - expected: []*Node{NumberNode("", 42)}, - }, - { - name: "wildcard on object", - args: pathData{ - node: ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - path: []string{"$", "*"}, - }, - expected: []*Node{NumberNode("", 42), NumberNode("", 43)}, - }, - { - name: "array slice", - args: pathData{ - node: ArrayNode("", []*Node{ - NumberNode("", 1), - NumberNode("", 2), - NumberNode("", 3), - NumberNode("", 4), - }), - path: []string{"$", "1:3"}, - }, - expected: []*Node{NumberNode("", 2), NumberNode("", 3)}, - }, - { - name: "array slice with step", - args: pathData{ - node: ArrayNode("", []*Node{ - NumberNode("", 1), - NumberNode("", 2), - NumberNode("", 3), - NumberNode("", 4), - NumberNode("", 5), - }), - path: []string{"$", "0:5:2"}, - }, - expected: []*Node{ - NumberNode("", 1), - NumberNode("", 3), - NumberNode("", 5), - }, - }, - { - name: "array slice with negative index and step", - args: pathData{ - node: ArrayNode("", []*Node{ - NumberNode("", 1), - NumberNode("", 2), - NumberNode("", 3), - NumberNode("", 4), - NumberNode("", 5), - }), - path: []string{"$", "-3:-1:1"}, - }, - expected: []*Node{ - NumberNode("", 3), - NumberNode("", 4), - }, - }, - { - name: "basic recursive descent path", - args: pathData{ - node: ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - path: []string{"$", ".."}, - }, - expected: []*Node{ - ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - NumberNode("", 42), - NumberNode("", 43), - }, - }, - { - name: "recursive descent with wildcard", - args: pathData{ - node: ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - path: []string{"$", "..", "*"}, - }, - expected: []*Node{ - ObjectNode("", map[string]*Node{ - "key1": NumberNode("", 42), - "key2": NumberNode("", 43), - }), - NumberNode("", 42), - NumberNode("", 41), - }, - }, + for i, node := range result { + expectedNorm := normalizeJSON(tt.expected[i]) + resultNorm := normalizeJSON(node.String()) + if resultNorm != expectedNorm { + t.Errorf("Path %q: expected result %q, got %q", tt.path, expectedNorm, resultNorm) + } + } } +} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, err := applyPath(tt.args.node, tt.args.path) - if (err != nil) != tt.isErr { - t.Errorf("#1 expected error %v, got %v", tt.isErr, err) - } - }) +// normalizeJSON removes all whitespace outside of quoted text. +func normalizeJSON(s string) string { + var sb strings.Builder + inQuotes := false + for i := 0; i < len(s); i++ { + c := s[i] + if c == '"' { + inQuotes = !inQuotes + sb.WriteByte(c) + } else if inQuotes { + sb.WriteByte(c) + } else if !inQuotes && !unicode.IsSpace(rune(c)) { + sb.WriteByte(c) + } } + return sb.String() } func sliceEqual(a, b []string) bool { if len(a) != len(b) { return false } - for i, v := range a { if v != b[i] { return false } } - return true } -func fullPath(array []*Node) string { - return sliceString(Paths(array)) -} - func sliceString(array []string) string { return "[" + strings.Join(array, ", ") + "]" } - -func (n *Node) Equals(other *Node) bool { - // Compare the values of n and other - return n.value == other.value && n.nodeType == other.nodeType -} diff --git a/gnovm/tests/files/json0.gno b/gnovm/tests/files/json0.gno deleted file mode 100644 index b77ef0e0e63..00000000000 --- a/gnovm/tests/files/json0.gno +++ /dev/null @@ -1,151 +0,0 @@ -package main - -import ( - "gno.land/p/demo/json" - "gno.land/p/demo/ufmt" -) - -func main() { - data := []byte(`{ - "store": { - "book": [ - { - "category": "reference", - "author": "Nigel Rees", - "title": "Sayings of the Century", - "price": 8.95 - }, - { - "category": "fiction", - "author": "Herman Melville", - "title": "Moby Dick", - "isbn": "0-553-21311-3", - "price": 8.99 - }, - { - "category": "fiction", - "author": "J.R.R. Tolkien", - "title": "The Lord of the Rings", - "isbn": "0-395-19395-8", - "price": 22.99 - } - ], - "bicycle": { - "color": "red", - "price": 19.95 - } - }, - "expensive": 10 - }`) - - paths := []string{ - "$.store.*", // All direct properties of `store` (not recursive) - "$.store.bicycle.color", // The color of the bicycle in the store (result: red) - "$.store.book[*]", // All books in the store - "$.store.book[0].title", // The title of the first book - - "$.store..price", // The prices of all items in the store - "$..price", // Result: [8.95, 8.99, 22.99, 19.95] - "$..book[*].title", // The titles of all books in the store - "$..book[0]", // The first book - } - - for _, path := range paths { - result, err := json.Path(data, path) - if err != nil { - ufmt.Println(err) - return - } - - ufmt.Println("Path:", path) - - for _, node := range result { - ufmt.Println(node.String()) - } - - ufmt.Println() - } -} - -// output: -// -// Path: $.store.* -// { -// "color": "red", -// "price": 19.95 -// } -// [ -// { -// "category": "reference", -// "author": "Nigel Rees", -// "title": "Sayings of the Century", -// "price": 8.95 -// }, -// { -// "category": "fiction", -// "author": "Herman Melville", -// "title": "Moby Dick", -// "isbn": "0-553-21311-3", -// "price": 8.99 -// }, -// { -// "category": "fiction", -// "author": "J.R.R. Tolkien", -// "title": "The Lord of the Rings", -// "isbn": "0-395-19395-8", -// "price": 22.99 -// } -// ] -// -// Path: $.store.bicycle.color -// "red" -// -// Path: $.store.book[*] -// { -// "category": "reference", -// "author": "Nigel Rees", -// "title": "Sayings of the Century", -// "price": 8.95 -// } -// { -// "category": "fiction", -// "author": "Herman Melville", -// "title": "Moby Dick", -// "isbn": "0-553-21311-3", -// "price": 8.99 -// } -// { -// "category": "fiction", -// "author": "J.R.R. Tolkien", -// "title": "The Lord of the Rings", -// "isbn": "0-395-19395-8", -// "price": 22.99 -// } -// -// Path: $.store.book[0].title -// "Sayings of the Century" -// -// Path: $.store..price -// 19.95 -// 8.95 -// 8.99 -// 22.99 -// -// Path: $..price -// 19.95 -// 8.95 -// 8.99 -// 22.99 -// -// Path: $..book[*].title -// "Sayings of the Century" -// "Moby Dick" -// "The Lord of the Rings" -// -// Path: $..book[0] -// { -// "category": "reference", -// "author": "Nigel Rees", -// "title": "Sayings of the Century", -// "price": 8.95 -// }