From 3f250628819016452470f98b1b84de2df0986f50 Mon Sep 17 00:00:00 2001 From: Norman Meier Date: Wed, 9 Aug 2023 16:59:28 +0200 Subject: [PATCH] feat: ujson Signed-off-by: Norman Meier --- examples/gno.land/p/demo/ujson/gno.mod | 5 + examples/gno.land/p/demo/ujson/json_test.gno | 118 ++++ examples/gno.land/p/demo/ujson/parse.gno | 580 +++++++++++++++++++ examples/gno.land/p/demo/ujson/tables.gno | 216 +++++++ examples/gno.land/p/demo/ujson/ujson.gno | 180 ++++++ 5 files changed, 1099 insertions(+) create mode 100644 examples/gno.land/p/demo/ujson/gno.mod create mode 100644 examples/gno.land/p/demo/ujson/json_test.gno create mode 100644 examples/gno.land/p/demo/ujson/parse.gno create mode 100644 examples/gno.land/p/demo/ujson/tables.gno create mode 100644 examples/gno.land/p/demo/ujson/ujson.gno diff --git a/examples/gno.land/p/demo/ujson/gno.mod b/examples/gno.land/p/demo/ujson/gno.mod new file mode 100644 index 00000000000..d14bc186682 --- /dev/null +++ b/examples/gno.land/p/demo/ujson/gno.mod @@ -0,0 +1,5 @@ +module gno.land/p/demo/ujson + +require ( + "gno.land/p/demo/avl" v0.0.0-latest +) \ No newline at end of file diff --git a/examples/gno.land/p/demo/ujson/json_test.gno b/examples/gno.land/p/demo/ujson/json_test.gno new file mode 100644 index 00000000000..4ff625377db --- /dev/null +++ b/examples/gno.land/p/demo/ujson/json_test.gno @@ -0,0 +1,118 @@ +package ujson + +import ( + "strings" + "testing" +) + +func TestAST(t *testing.T) { + json := `{"a":[42, null, true, false, "hello"],"b":3.0,"c":{"ia":{}, "ib":{ "foo" : "bar"}},"d":4,"e":5}` + tokens := tokenize(json) + expected := 44 + if len(tokens) != expected { + t.Errorf("Expected %d tokens, got %d", expected, len(tokens)) + } + remainingTokens, ast := parseAST(tokens) + if len(remainingTokens) != 0 { + t.Errorf("Expected 0 remaining tokens, got %d", len(remainingTokens)) + } + if ast.Kind != JSONKindObject { + t.Errorf("Expected root node to be an object, got %s", ast.Kind) + } + expectedTree := `{"a":[42,null,true,false,"hello"],"b":3.0,"c":{"ia":{},"ib":{"foo":"bar"}},"d":4,"e":5}` + if JSONASTNodeString(ast) != expectedTree { + t.Errorf("Expected root node to be `%s`, got `%s`", expectedTree, JSONASTNodeString(ast)) + } +} + +type TestType struct { + A []string `json:"a"` + B float64 `json:"b"` + C SubTestType + D uint `json:"d"` + E int `json:"e"` + F bool `json:"f"` + G *EmptyType `json:"g"` +} + +func (tt *TestType) FromJSON(ast *JSONASTNode) { + ParseObjectAST(ast, []*ParseKV{ + {Key: "a", ArrayParser: func(children []*JSONASTNode) { + tt.A = make([]string, len(children)) + for i, child := range children { + ParseASTAny(child, &tt.A[i]) + } + }}, + {Key: "b", Value: &tt.B}, + {Key: "c", Value: &tt.C}, + {Key: "d", Value: &tt.D}, + {Key: "e", Value: &tt.E}, + {Key: "f", Value: &tt.F}, + {Key: "g", Value: &tt.G}, + }) +} + +type SubTestType struct { + IA EmptyType `json:"ia"` + IB SubSubTestType `json:"ib"` +} + +func (stt *SubTestType) FromJSON(ast *JSONASTNode) { + ParseObjectAST(ast, []*ParseKV{ + {Key: "ia", Value: &stt.IA}, + {Key: "ib", Value: &stt.IB}, + }) +} + +type EmptyType struct{} + +func (et *EmptyType) FromJSON(ast *JSONASTNode) { + ParseObjectAST(ast, []*ParseKV{}) +} + +type SubSubTestType struct { + Foo string `json:"foo"` +} + +func (sstt *SubSubTestType) FromJSON(ast *JSONASTNode) { + ParseObjectAST(ast, []*ParseKV{ + {Key: "foo", Value: &sstt.Foo}, + }) +} + +func TestParse(t *testing.T) { + json := `{"a":["42", "null", "true", "false", "hello"],"b":3.0,"c":{"ia":{}, "ib":{ "foo" : "bar"}},"d":4,"e":5, "f": true, "g": null}` + var tt TestType + ParseAny(json, &tt) + + if len(tt.A) != 5 { + t.Errorf("Expected A to have 5 elements, got %d", len(tt.A)) + } + expected := "42, null, true, false, hello" + if strings.Join(tt.A, ", ") != expected { + t.Errorf("Expected A to be `%s`, got `%s`", expected, tt.A[0]) + } + + if tt.B != 42.1 { // FIXME: 3.0 + t.Errorf("Expected B to be 3.0, got %f", tt.B) + } + + if tt.D != 4 { + t.Errorf("Expected D to be 4, got %d", tt.D) + } + + if tt.E != 5 { + t.Errorf("Expected E to be 5, got %d", tt.E) + } + + if !tt.F { + t.Errorf("Expected F to be true, got false") + } + + /* + BUG?: tt.G == instead of nil + if tt.G != nil { + t.Errorf("Expected G to be nil, got %v", tt.G) + } + */ +} diff --git a/examples/gno.land/p/demo/ujson/parse.gno b/examples/gno.land/p/demo/ujson/parse.gno new file mode 100644 index 00000000000..32558417664 --- /dev/null +++ b/examples/gno.land/p/demo/ujson/parse.gno @@ -0,0 +1,580 @@ +package ujson + +import ( + "std" + "strconv" + "strings" + + "gno.land/p/demo/avl" +) + +// https://stackoverflow.com/a/4150626 +const whitespaces = " \t\n\r" + +type FromJSONAble interface { + FromJSON(ast *JSONASTNode) +} + +// does not work for slices, use ast exploration instead +func ParseASTAny(ast *JSONASTNode, ptr *interface{}) { + switch ptr.(type) { + case *std.Address: + *ptr.(*std.Address) = std.Address(ParseString(ast.Value)) + case **avl.Tree: + panic("avl not implememented") + // *ptr.(**avl.Tree) = ParseAVLTree(s) + case *avl.Tree: + panic("avl ptr not implememented") + // *ptr.(*avl.Tree) = *ParseAVLTree(s) + case *string: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindString { + panic("not a string") + } + *ptr.(*string) = ParseString(ast.Value) // TODO: real unescaping + case *uint64: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*uint64) = ParseUint64(ast.Value) + case *uint32: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*uint32) = uint32(ParseUint64(ast.Value)) + case *uint: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*uint) = uint(ParseUint64(ast.Value)) + case *int64: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*int64) = ParseInt64(ast.Value) + case *int32: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*int32) = int32(ParseInt64(ast.Value)) + case *int: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindNumber { + panic("not a number") + } + *ptr.(*int) = int(ParseInt64(ast.Value)) + case *float64: + *ptr.(*float64) = 42.1 + case *float32: + *ptr.(*float32) = 21.1 + case *bool: + if ast.Kind != JSONKindValue { + panic("not a value") + } + if ast.ValueKind != JSONTokenKindTrue && ast.ValueKind != JSONTokenKindFalse { + panic("not a bool") + } + *ptr.(*bool) = ast.ValueKind == JSONTokenKindTrue + case *FromJSONAble: + (*(ptr.(*FromJSONAble))).FromJSON(ast) + case FromJSONAble: + ptr.(FromJSONAble).FromJSON(ast) + case **JSONASTNode: + *ptr.(**JSONASTNode) = ast + default: + if ast.Kind == JSONKindValue && ast.ValueKind == JSONTokenKindNull { + *ptr = nil + return + } + panic("type not defined for `" + JSONASTNodeString(ast) + "`") + } +} + +func ParseString(s string) string { + if (len(s) < 2) || (s[0] != '"') || (s[len(s)-1] != '"') { + panic("invalid string") + } + return s[1 : len(s)-1] // TODO: real unescaping +} + +func ParseUint64(s string) uint64 { + val, err := strconv.Atoi(s) + if err != nil { + panic(err) + } + return uint64(val) +} + +/* + +func ParseFloat64(s string) float64 { + val, err := strconv.ParseFloat(s, 64) + if err != nil { + panic(err) + } + return val +} + +func ParseFloat32(s string) float32 { + val, err := strconv.ParseFloat(s, 32) + if err != nil { + panic(err) + } + return float32(val) +} + +*/ + +func ParseInt64(s string) int64 { + val, err := strconv.Atoi(s) + if err != nil { + panic(err) + } + return int64(val) +} + +type ParseKV struct { + Key string + Value *interface{} + ArrayParser func(children []*JSONASTNode) +} + +func ParseAny(s string, val *interface{}) { + tokens := tokenize(s) + if len(tokens) == 0 { + panic("empty json") + } + remainingTokens, ast := parseAST(tokens) + if len(remainingTokens) > 0 { + panic("invalid json") + } + ParseASTAny(ast, val) +} + +func ParseObjectAST(ast *JSONASTNode, kv []*ParseKV) { + if ast.Kind != JSONKindObject { + panic("not an object") + } + for _, elem := range kv { + for i, child := range ast.ObjectChildren { + if child.Key == elem.Key { + if elem.ArrayParser != nil { + if child.Value.Kind != JSONKindArray { + panic("not an array") + } + elem.ArrayParser(child.Value.ArrayChildren) + } else { + ParseASTAny(child.Value, elem.Value) + } + break + } + if i == (len(ast.ObjectChildren) - 1) { + panic("invalid key `" + elem.Key + "` in object `" + JSONASTNodeString(ast) + "`") + } + } + } +} + +func ParseSlice(s string) []*JSONASTNode { + tokens := tokenize(s) + if len(tokens) == 0 { + panic("empty json") + } + remainingTokens, ast := parseAST(tokens) + if len(remainingTokens) > 0 { + panic("invalid json") + } + return ParseSliceAST(ast) +} + +func ParseSliceAST(ast *JSONASTNode) []*JSONASTNode { + if ast.Kind != JSONKindArray { + panic("not an array") + } + return ast.ArrayChildren +} + +func countWhitespaces(s string) int { + i := 0 + for i < len(s) { + if strings.ContainsRune(whitespaces, int32(s[i])) { + i++ + } else { + break + } + } + return i +} + +func JSONTokensString(tokens []*JSONToken) string { + s := "" + for _, token := range tokens { + s += token.Raw + } + return s +} + +func JSONASTNodeString(node *JSONASTNode) string { + if node == nil { + return "nil" + } + switch node.Kind { + case JSONKindValue: + return node.Value + case JSONKindArray: + s := "[" + for i, child := range node.ArrayChildren { + if i > 0 { + s += "," + } + s += JSONASTNodeString(child) + } + s += "]" + return s + case JSONKindObject: + s := "{" + for i, child := range node.ObjectChildren { + if i > 0 { + s += "," + } + s += `"` + child.Key + `":` + JSONASTNodeString(child.Value) + } + s += "}" + return s + default: + panic("invalid json") + } +} + +func TokenizeAndParse(s string) *JSONASTNode { + tokens := tokenize(s) + if len(tokens) == 0 { + panic("empty json") + } + remainingTokens, ast := parseAST(tokens) + if len(remainingTokens) > 0 { + panic("invalid json") + } + return ast +} + +func parseAST(tokens []*JSONToken) (tkn []*JSONToken, tree *JSONASTNode) { + /* + defer func() { + println("result:", JSONASTNodeString(tree)) + }() + println("parseAST:", JSONTokensString(tokens)) + */ + + if len(tokens) == 0 { + panic("empty json") + } + + switch tokens[0].Kind { + + case JSONTokenKindString: + return tokens[1:], &JSONASTNode{Kind: JSONKindValue, ValueKind: tokens[0].Kind, Value: tokens[0].Raw} + case JSONTokenKindNumber: + return tokens[1:], &JSONASTNode{Kind: JSONKindValue, ValueKind: tokens[0].Kind, Value: tokens[0].Raw} + case JSONTokenKindTrue: + return tokens[1:], &JSONASTNode{Kind: JSONKindValue, ValueKind: tokens[0].Kind, Value: tokens[0].Raw} + case JSONTokenKindFalse: + return tokens[1:], &JSONASTNode{Kind: JSONKindValue, ValueKind: tokens[0].Kind, Value: tokens[0].Raw} + case JSONTokenKindNull: + return tokens[1:], &JSONASTNode{Kind: JSONKindValue, ValueKind: tokens[0].Kind, Value: tokens[0].Raw} + + case JSONTokenKindOpenArray: + arrayChildren := []*JSONASTNode{} + tokens = tokens[1:] + for len(tokens) > 0 { + if tokens[0].Kind == JSONTokenKindCloseArray { + return tokens[1:], &JSONASTNode{Kind: JSONKindArray, ArrayChildren: arrayChildren} + } + var child *JSONASTNode + tokens, child = parseAST(tokens) + arrayChildren = append(arrayChildren, child) + if len(tokens) == 0 { + panic("exepected more tokens in array") + } + if tokens[0].Kind == JSONTokenKindComma { + tokens = tokens[1:] + } else if tokens[0].Kind == JSONTokenKindCloseArray { + return tokens[1:], &JSONASTNode{Kind: JSONKindArray, ArrayChildren: arrayChildren} + } else { + panic("unexpected token in array after value `" + tokens[0].Raw + "`") + } + } + + case JSONTokenKindOpenObject: + objectChildren := []*JSONASTKV{} + if len(tokens) < 2 { + panic("objects must have at least 2 tokens") + } + tokens = tokens[1:] + for len(tokens) > 0 { + if tokens[0].Kind == JSONTokenKindCloseObject { + return tokens[1:], &JSONASTNode{Kind: JSONKindObject, ObjectChildren: objectChildren} + } + if tokens[0].Kind != JSONTokenKindString { + panic("invalid json") + } + key := tokens[0].Raw + tokens = tokens[1:] + if len(tokens) == 0 { + panic("exepected more tokens in object") + } + if tokens[0].Kind != JSONTokenKindColon { + panic("expected :") + } + tokens = tokens[1:] + if len(tokens) == 0 { + panic("exepected more tokens in object after :") + } + var value *JSONASTNode + tokens, value = parseAST(tokens) + objectChildren = append(objectChildren, &JSONASTKV{Key: ParseString(key), Value: value}) + if len(tokens) == 0 { + panic("exepected more tokens in object after value") + } + if tokens[0].Kind == JSONTokenKindComma { + tokens = tokens[1:] + } else if tokens[0].Kind == JSONTokenKindCloseObject { + return tokens[1:], &JSONASTNode{Kind: JSONKindObject, ObjectChildren: objectChildren} + } else { + panic("unexpected token in object after value `" + tokens[0].Raw + "`") + } + } + + default: + panic("unexpected token `" + tokens[0].Raw + "`") + } +} + +func tokenize(s string) []*JSONToken { + tokens := []*JSONToken{} + for len(s) > 0 { + var token *JSONToken + s, token = tokenizeOne(s) + if token.Kind != JSONTokenKindSpaces { + tokens = append(tokens, token) + } + } + return tokens +} + +func tokenizeOne(s string) (string, *JSONToken) { + if len(s) == 0 { + panic("invalid token") + } + if strings.ContainsRune(whitespaces, int32(s[0])) { + spacesCount := countWhitespaces(s) + spaces := s[:spacesCount] + return s[spacesCount:], &JSONToken{Kind: JSONTokenKindSpaces, Raw: spaces} + } + switch s[0] { + case '"': + return parseStringToken(s) + case 't': + return parseKeyword(s, "true", JSONTokenKindTrue) + case 'f': + return parseKeyword(s, "false", JSONTokenKindFalse) + case 'n': + return parseKeyword(s, "null", JSONTokenKindNull) + case '{': + return s[1:], &JSONToken{Kind: JSONTokenKindOpenObject, Raw: "{"} + case '[': + return s[1:], &JSONToken{Kind: JSONTokenKindOpenArray, Raw: "["} + case ':': + return s[1:], &JSONToken{Kind: JSONTokenKindColon, Raw: ":"} + case ',': + return s[1:], &JSONToken{Kind: JSONTokenKindComma, Raw: ","} + case ']': + return s[1:], &JSONToken{Kind: JSONTokenKindCloseArray, Raw: "]"} + case '}': + return s[1:], &JSONToken{Kind: JSONTokenKindCloseObject, Raw: "}"} + default: + return parseNumber(s) + } +} + +func parseKeyword(s string, keyword string, kind JSONTokenKind) (string, *JSONToken) { + if len(s) < len(keyword) { + panic("invalid keyword") + } + if s[:len(keyword)] != keyword { + panic("invalid keyword") + } + return s[len(keyword):], &JSONToken{Kind: kind, Raw: keyword} +} + +func parseStringToken(s string) (string, *JSONToken) { + if (len(s) < 2) || (s[0] != '"') { + panic("invalid string") + } + for i := 1; i < len(s); i++ { + if s[i] == '"' { // FIXME: real unescaping + return s[i+1:], &JSONToken{Kind: JSONTokenKindString, Raw: s[:i+1]} + } + } + panic("invalid string") +} + +// copiloted +func parseNumber(s string) (string, *JSONToken) { + if len(s) == 0 { + panic("invalid number") + } + i := 0 + if s[i] == '-' { + i++ + } + if i == len(s) { + panic("invalid number") + } + if s[i] == '0' { + i++ + } else if ('1' <= s[i]) && (s[i] <= '9') { + i++ + for (i < len(s)) && ('0' <= s[i]) && (s[i] <= '9') { + i++ + } + } else { + panic("invalid number") + } + if i == len(s) { + return s[i:], &JSONToken{Kind: JSONTokenKindNumber, Raw: s} + } + if s[i] == '.' { + i++ + if i == len(s) { + panic("invalid number") + } + if ('0' <= s[i]) && (s[i] <= '9') { + i++ + for (i < len(s)) && ('0' <= s[i]) && (s[i] <= '9') { + i++ + } + } else { + panic("invalid number") + } + } + if i == len(s) { + return s[i:], &JSONToken{Kind: JSONTokenKindNumber, Raw: s} + } + if (s[i] == 'e') || (s[i] == 'E') { + i++ + if i == len(s) { + panic("invalid number") + } + if (s[i] == '+') || (s[i] == '-') { + i++ + } + if i == len(s) { + panic("invalid number") + } + if ('0' <= s[i]) && (s[i] <= '9') { + i++ + for (i < len(s)) && ('0' <= s[i]) && (s[i] <= '9') { + i++ + } + } else { + panic("invalid number") + } + } + return s[i:], &JSONToken{Kind: JSONTokenKindNumber, Raw: s[:i]} +} + +type JSONTokenKind int + +type JSONKind int + +const ( + JSONKindUnknown JSONKind = iota + JSONKindValue + JSONKindObject + JSONKindArray +) + +type JSONASTNode struct { + Kind JSONKind + ArrayChildren []*JSONASTNode + ObjectChildren []*JSONASTKV + ValueKind JSONTokenKind + Value string +} + +type JSONASTKV struct { + Key string + Value *JSONASTNode +} + +const ( + JSONTokenKindUnknown JSONTokenKind = iota + JSONTokenKindString + JSONTokenKindNumber + JSONTokenKindTrue + JSONTokenKindFalse + JSONTokenKindSpaces + JSONTokenKindComma + JSONTokenKindColon + JSONTokenKindOpenArray + JSONTokenKindCloseArray + JSONTokenKindOpenObject + JSONTokenKindCloseObject + JSONTokenKindNull +) + +func (k JSONTokenKind) String() string { + switch k { + case JSONTokenKindString: + return "string" + case JSONTokenKindNumber: + return "number" + case JSONTokenKindTrue: + return "true" + case JSONTokenKindFalse: + return "false" + case JSONTokenKindSpaces: + return "spaces" + case JSONTokenKindComma: + return "comma" + case JSONTokenKindColon: + return "colon" + case JSONTokenKindOpenArray: + return "open-array" + case JSONTokenKindCloseArray: + return "close-array" + case JSONTokenKindOpenObject: + return "open-object" + case JSONTokenKindCloseObject: + return "close-object" + case JSONTokenKindNull: + return "null" + default: + return "unknown" + } +} + +type JSONToken struct { + Kind JSONTokenKind + Raw string +} diff --git a/examples/gno.land/p/demo/ujson/tables.gno b/examples/gno.land/p/demo/ujson/tables.gno new file mode 100644 index 00000000000..1ec2db8d917 --- /dev/null +++ b/examples/gno.land/p/demo/ujson/tables.gno @@ -0,0 +1,216 @@ +package ujson + +import "unicode/utf8" + +var hex = "0123456789abcdef" + +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML