feat: support streaming IO (#225)

* feat (decoder): support streaming IO

* opt: fine tune buffer size factors

* feat (encoder): support streaming IO

* doc: refactor format and add comments

* fix: io.EOF dead loop

* build: adjust CI files

* doc: update README.md

.github/workflows/push-check-benchmark.yml → .github/workflows/benchmark.yml

@@ -1,4 +1,4 @@
-name: Pull Request Check
+name: Pull Request Benchmark
 
 on: pull_request
 

.github/workflows/push-check-go118.yml → .github/workflows/push-check-linux-go118.yml

@@ -21,7 +21,7 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Unit Test
-        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race -covermode=atomic -coverprofile=coverage.out ./...
+        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race ./...
 
       - name: Generic Test
-        run: go test -v -gcflags=-d=checkptr=0 -race -covermode=atomic ./generic_test
+        run: go test -v -gcflags=-d=checkptr=0 -race ./generic_test

.github/workflows/push-check-go117.yml → .github/workflows/push-check-linux.yml

@@ -1,17 +1,20 @@
-name: Push Check Go1.17
+name: Push Check Linux
 
 on: push
 
 jobs:
   build:
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x, 1.17.x]
     runs-on: self-hosted
     steps:
       - uses: actions/checkout@v2
 
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.17
+          go-version: ${{ matrix.go-version }}
 
       - uses: actions/cache@v2
         with:
@@ -21,4 +24,4 @@ jobs:
             ${{ runner.os }}-go-
 
       - name: Unit Test
-        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 -race -covermode=atomic -coverprofile=coverage.out ./...
+        run: GOMAXPROCS=4 go test -v -gcflags=-d=checkptr=0 ./...

.github/workflows/push-check-windows.yml

@@ -0,0 +1,26 @@
+# name: Push Check Go Windows
+
+# on: push
+
+# jobs:
+#   build:
+#     strategy:
+#       matrix:
+#         go-version: [1.15.x, 1.16.x, 1.17.x, 1.18.x]
+#         os: [windows-latest]
+#     runs-on: ${{ matrix.os }}
+#     steps:
+#       - uses: actions/checkout@v2
+
+#       - name: Set up Go
+#         uses: actions/setup-go@v3
+#         with:
+#           go-version: ${{ matrix.go-version }}
+
+#       - name: Unit Test
+#         run: go test -v -gcflags -d=checkptr=0 -covermode atomic -coverprofile coverage.out ./...
+#         env:
+#           GOMAXPROCS: 4
+
+#       - name: Generic Test
+#         run: go test -v -gcflags -d=checkptr=0 -covermode atomic ./generic_test

README.md

@@ -97,9 +97,36 @@ var data YourSchema
 // Marshal
 output, err := sonic.Marshal(&data) 
 // Unmarshal
-err := sonic.Unmarshal(output, &data) 
+err := sonic.Unmarshal(output, &data)
  ```
 
+### Streaming IO
+Sonic supports to decode json from `io.Reader` or encode objects into `io.Writer`, aiming at handling multiple values as well as reducing memory consuming.
+- encoder
+```go
+import "github.com/bytedance/sonic/encoder"
+
+var o1 =  map[string]interface{}{
+         "a": "b"
+}
+var o2 = 1
+var w = bytes.NewBuffer(nil)
+var enc = encoder.NewStreamEncoder(w)
+enc.Encode(o)
+println(w1.String()) // "{\"a\":\"b\"}\n1"
+```
+- decoder
+```go
+import "github.com/bytedance/sonic/decoder"
+
+var o =  map[string]interface{}{}
+var r = strings.NewReader(`{"a":"b"}{"1":"2"}`)
+var dec = decoder.NewStreamDecoder(r)
+dec.Decode(&o)
+dec.Decode(&o)
+fmt.Printf("%+v", o) // map[1:2 a:b]
+```
+
 ### Use Number/Use Int64
  ```go
 import "github.com/bytedance/sonic/decoder"

bench.sh

@@ -4,17 +4,15 @@ pwd=$(pwd)
 export SONIC_NO_ASYNC_GC=1
 
 cd $pwd/encoder
-go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkEncoder_Generic_Sonic|BenchmarkEncoder_Generic_Sonic_Fast|BenchmarkEncoder_Generic_JsonIter|BenchmarkEncoder_Generic_GoJson|BenchmarkEncoder_Generic_StdLib|BenchmarkEncoder_Binding_Sonic|BenchmarkEncoder_Binding_Sonic_Fast|BenchmarkEncoder_Binding_JsonIter|BenchmarkEncoder_Binding_GoJson|BenchmarkEncoder_Binding_StdLib|BenchmarkEncoder_Parallel_Generic_Sonic|BenchmarkEncoder_Parallel_Generic_Sonic_Fast|BenchmarkEncoder_Parallel_Generic_JsonIter|BenchmarkEncoder_Parallel_Generic_GoJson|BenchmarkEncoder_Parallel_Generic_StdLib|BenchmarkEncoder_Parallel_Binding_Sonic|BenchmarkEncoder_Parallel_Binding_Sonic_Fast|BenchmarkEncoder_Parallel_Binding_JsonIter|BenchmarkEncoder_Parallel_Binding_GoJson|BenchmarkEncoder_Parallel_Binding_StdLib)$"
+go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkEncoder_.*)$"
 
 cd $pwd/decoder
-go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkDecoder_Generic_Sonic|BenchmarkDecoder_Generic_Sonic_Fast|BenchmarkDecoder_Generic_StdLib|BenchmarkDecoder_Generic_JsonIter|BenchmarkDecoder_Generic_GoJson|BenchmarkDecoder_Binding_Sonic|BenchmarkDecoder_Binding_Sonic_Fast|BenchmarkDecoder_Binding_StdLib|BenchmarkDecoder_Binding_JsonIter|BenchmarkDecoder_Binding_GoJson|BenchmarkDecoder_Parallel_Generic_Sonic|BenchmarkDecoder_Parallel_Generic_Sonic_Fast|BenchmarkDecoder_Parallel_Generic_StdLib|BenchmarkDecoder_Parallel_Generic_JsonIter|BenchmarkDecoder_Parallel_Generic_GoJson|BenchmarkDecoder_Parallel_Binding_Sonic|BenchmarkDecoder_Parallel_Binding_Sonic_Fast|BenchmarkDecoder_Parallel_Binding_StdLib|BenchmarkDecoder_Parallel_Binding_JsonIter|BenchmarkDecoder_Parallel_Binding_GoJson)$"
+go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkDecoder_.*)$"
 
 cd $pwd/ast
-go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkGetOne_Sonic|BenchmarkGetOne_Gjson|BenchmarkGetOne_Jsoniter|BenchmarkGetOne_Parallel_Sonic|BenchmarkGetOne_Parallel_Gjson|BenchmarkGetOne_Parallel_Jsoniter|BenchmarkSetOne_Sonic|BenchmarkSetOne_Sjson|BenchmarkSetOne_Jsoniter|BenchmarkSetOne_Parallel_Sonic|BenchmarkSetOne_Parallel_Sjson|BenchmarkSetOne_Parallel_Jsoniter)$"
+go test -benchmem -run=^$ -benchtime=100000x -bench "^(BenchmarkGet.*|BenchmarkEncode.*)$"
 
-go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_Sonic|BenchmarkParser_Gjson|BenchmarkParser_JsonIter|BenchmarkParser_Parallel_Sonic|BenchmarkParser_Parallel_Gjson|BenchmarkParser_Parallel_StdLib|BenchmarkParser_Parallel_JsonIter|BenchmarkParseOne_Sonic|BenchmarkParseOne_Gjson|BenchmarkParseOne_Jsoniter|BenchmarkParseOne_Parallel_Sonic|BenchmarkParseOne_Parallel_Gjson|BenchmarkParseOne_Parallel_Jsoniter|BenchmarkParseSeven_Sonic|BenchmarkParseSeven_Gjson|BenchmarkParseSeven_Jsoniter|BenchmarkParseSeven_Parallel_Sonic|BenchmarkParseSeven_Parallel_Gjson|BenchmarkParseSeven_Parallel_Jsoniter)$"
-
-go test -benchmem -run=^$ -benchtime=100000x -bench '^(BenchmarkEncodeRaw|BenchmarkEncodeSkip|BenchmarkEncodeLoad)$'
+go test -benchmem -run=^$ -benchtime=10000x -bench "^(BenchmarkParser_.*)$"
 
 go test -benchmem -run=^$ -benchtime=10000000x -bench "^(BenchmarkNodeGetByPath|BenchmarkStructGetByPath|BenchmarkNodeIndex|BenchmarkStructIndex|BenchmarkSliceIndex|BenchmarkMapIndex|BenchmarkNodeGet|BenchmarkSliceGet|BenchmarkMapGet|BenchmarkNodeSet|BenchmarkMapSet|BenchmarkNodeSetByIndex|BenchmarkSliceSetByIndex|BenchmarkStructSetByIndex|BenchmarkNodeUnset|BenchmarkMapUnset|BenchmarkNodUnsetByIndex|BenchmarkSliceUnsetByIndex|BenchmarkNodeAdd|BenchmarkSliceAdd|BenchmarkMapAdd)$"
 

decoder/decoder.go

@@ -54,6 +54,12 @@ func (self *Decoder) Pos() int {
     return self.i
 }
 
+func (self *Decoder) Reset(s string) {
+    self.s = s
+    self.i = 0
+    // self.f = 0
+}
+
 // Decode parses the JSON-encoded data from current position and stores the result
 // in the value pointed to by val.
 func (self *Decoder) Decode(val interface{}) error {
@@ -75,9 +81,6 @@ func (self *Decoder) Decode(val interface{}) error {
     nb, err := decodeTypedPointer(self.s, self.i, etp, vp, sb, self.f)
 
     /* return the stack back */
-    if err != nil {
-        resetStack(sb)
-    }
     self.i = nb
     freeStack(sb)
 

decoder/errors.go

@@ -32,6 +32,7 @@ type SyntaxError struct {
     Pos  int
     Src  string
     Code types.ParsingError
+    Msg  string
 }
 
 func (self SyntaxError) Error() string {
@@ -73,13 +74,20 @@ func (self SyntaxError) Description() string {
     return fmt.Sprintf(
         "Syntax error at index %d: %s\n\n\t%s\n\t%s^%s\n",
         self.Pos,
-        self.Code.Message(),
+        self.Message(),
         self.Src[p:q],
         strings.Repeat(".", x),
         strings.Repeat(".", y),
     )
 }
 
+func (self SyntaxError) Message() string {
+    if self.Msg == "" {
+        return self.Code.Message()
+    }
+    return self.Msg
+}
+
 func clamp_zero(v int) int {
     if v < 0 {
         return 0