From 5e72c96cf406ccad8b21e769ee73ea624264e80b Mon Sep 17 00:00:00 2001 From: Phil Levchenko Date: Mon, 16 Apr 2018 03:45:10 +0300 Subject: [PATCH 1/2] Fix 3rd party package's dependencies --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 37fc9fd..ce62415 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,13 @@ -FROM golang:1.6 +FROM golang:1.7 RUN go get github.com/Jeffail/gabs RUN go get github.com/bitly/go-simplejson RUN go get github.com/pquerna/ffjson RUN go get github.com/antonholmquist/jason RUN go get github.com/mreiferson/go-ujson +RUN go get github.com/a8m/djson RUN go get -tags=unsafe -u github.com/ugorji/go/codec RUN go get github.com/mailru/easyjson WORKDIR /go/src/github.com/buger/jsonparser -ADD . /go/src/github.com/buger/jsonparser \ No newline at end of file +ADD . /go/src/github.com/buger/jsonparser From 435a087f734734bb73b2b86591dba94f646bfb13 Mon Sep 17 00:00:00 2001 From: Phil Levchenko Date: Mon, 16 Apr 2018 03:46:06 +0300 Subject: [PATCH 2/2] Add early termination from ArrayEach() callback Added ability to terminate ArrayEach() from within its callback, by setting error to some value. Previous implementation didn't make much sense, since error was always nil. --- README.md | 4 +-- benchmark/benchmark_large_payload_test.go | 4 +-- benchmark/benchmark_medium_payload_test.go | 8 +++--- parser.go | 12 ++++----- parser_test.go | 31 ++++++++++++++++++---- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index a9ee602..25de671 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,9 @@ If key data type do not match, it will return error. ### **`ArrayEach`** ```go -func ArrayEach(data []byte, cb func(value []byte, dataType jsonparser.ValueType, offset int, err error), keys ...string) +func ArrayEach(data []byte, cb func(value []byte, dataType jsonparser.ValueType, offset int, err *error), keys ...string) ``` -Needed for iterating arrays, accepts a callback function with the same return arguments as `Get`. +Used for iterating arrays, accepts a callback function with the same return arguments as `Get`, except for error. If error is set from within callback, ArrayEach() terminates immediately, returning the same error. ### **`ObjectEach`** ```go diff --git a/benchmark/benchmark_large_payload_test.go b/benchmark/benchmark_large_payload_test.go index 844ba51..9587bbc 100644 --- a/benchmark/benchmark_large_payload_test.go +++ b/benchmark/benchmark_large_payload_test.go @@ -23,12 +23,12 @@ import ( */ func BenchmarkJsonParserLarge(b *testing.B) { for i := 0; i < b.N; i++ { - jsonparser.ArrayEach(largeFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(largeFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err *error) { jsonparser.Get(value, "username") nothing() }, "users") - jsonparser.ArrayEach(largeFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(largeFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err *error) { jsonparser.GetInt(value, "id") jsonparser.Get(value, "slug") nothing() diff --git a/benchmark/benchmark_medium_payload_test.go b/benchmark/benchmark_medium_payload_test.go index 9ee8113..ec93b35 100644 --- a/benchmark/benchmark_medium_payload_test.go +++ b/benchmark/benchmark_medium_payload_test.go @@ -31,7 +31,7 @@ func BenchmarkJsonParserMedium(b *testing.B) { jsonparser.GetInt(mediumFixture, "person", "github", "followers") jsonparser.Get(mediumFixture, "company") - jsonparser.ArrayEach(mediumFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(mediumFixture, func(value []byte, dataType jsonparser.ValueType, offset int, err *error) { jsonparser.Get(value, "url") nothing() }, "person", "gravatar", "avatars") @@ -69,7 +69,7 @@ func BenchmarkJsonParserEachKeyManualMedium(b *testing.B) { case 2: // jsonparser.ParseString(value) case 3: - jsonparser.ArrayEach(value, func(avalue []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(value, func(avalue []byte, dataType jsonparser.ValueType, offset int, err *error) { jsonparser.Get(avalue, "url") }) } @@ -105,7 +105,7 @@ func BenchmarkJsonParserEachKeyStructMedium(b *testing.B) { json.Unmarshal(value, &data.Company) // we don't have a JSON -> map[string]interface{} function yet, so use standard encoding/json here case 3: var avatars []*CBAvatar - jsonparser.ArrayEach(value, func(avalue []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(value, func(avalue []byte, dataType jsonparser.ValueType, offset int, err *error) { url, _ := jsonparser.ParseString(avalue) avatars = append(avatars, &CBAvatar{Url: url}) }) @@ -141,7 +141,7 @@ func BenchmarkJsonParserObjectEachStructMedium(b *testing.B) { missing-- case bytes.Equal(k, gravatarKey): var avatars []*CBAvatar - jsonparser.ArrayEach(v, func(avalue []byte, dataType jsonparser.ValueType, offset int, err error) { + jsonparser.ArrayEach(v, func(avalue []byte, dataType jsonparser.ValueType, offset int, err *error) { url, _ := jsonparser.ParseString(avalue) avatars = append(avatars, &CBAvatar{Url: url}) }, "avatars") diff --git a/parser.go b/parser.go index 9e8b788..b75a6b5 100644 --- a/parser.go +++ b/parser.go @@ -283,7 +283,7 @@ func searchKeys(data []byte, keys ...string) int { var valueFound []byte var valueOffset int var curI = i - ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err *error) { if curIdx == aIdx { valueFound = value valueOffset = offset @@ -473,7 +473,7 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str level++ var curIdx int - arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) { + arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err *error) { if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 { for pi, p := range paths { if pIdxFlags&bitwiseFlags[pi+1] != 0 { @@ -875,8 +875,8 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType, return value, dataType, offset, endOffset, nil } -// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`. -func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err error), keys ...string) (offset int, err error) { +// ArrayEach is used for iterating arrays, accepts a callback function with the same return arguments as `Get`, except for error. If error is set from within callback, ArrayEach() terminates immediately, returning the same error. +func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int, err *error), keys ...string) (offset int, err error) { if len(data) == 0 { return -1, MalformedObjectError } @@ -926,11 +926,11 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int } if t != NotExist { - cb(v, t, offset+o-len(v), e) + cb(v, t, offset+o-len(v), &e) } if e != nil { - break + return offset, e } offset += o diff --git a/parser_test.go b/parser_test.go index ad6dfba..d174d63 100644 --- a/parser_test.go +++ b/parser_test.go @@ -2,6 +2,7 @@ package jsonparser import ( "bytes" + "errors" "fmt" _ "fmt" "reflect" @@ -12,7 +13,7 @@ import ( var activeTest = "" func toArray(data []byte) (result [][]byte) { - ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data, func(value []byte, dataType ValueType, offset int, err *error) { result = append(result, value) }) @@ -20,7 +21,7 @@ func toArray(data []byte) (result [][]byte) { } func toStringArray(data []byte) (result []string) { - ArrayEach(data, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(data, func(value []byte, dataType ValueType, offset int, err *error) { result = append(result, string(value)) }) @@ -1191,7 +1192,7 @@ func TestArrayEach(t *testing.T) { mock := []byte(`{"a": { "b":[{"x": 1} ,{"x":2},{ "x":3}, {"x":4} ]}}`) count := 0 - ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err error) { + ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err *error) { count++ switch count { @@ -1218,11 +1219,11 @@ func TestArrayEach(t *testing.T) { } func TestArrayEachEmpty(t *testing.T) { - funcError := func([]byte, ValueType, int, error) { t.Errorf("Run func not allow") } + funcError := func([]byte, ValueType, int, *error) { t.Errorf("Run func not allow") } type args struct { data []byte - cb func(value []byte, dataType ValueType, offset int, err error) + cb func(value []byte, dataType ValueType, offset int, err *error) keys []string } tests := []struct { @@ -1252,6 +1253,26 @@ func TestArrayEachEmpty(t *testing.T) { } } +func TestArrayEachEarlyTermination(t *testing.T) { + earlyTermError := errors.New("Early termination") + mock := []byte(`{"a":[{"x":1},{"x":2},{"x":3},{"x":4}]}`) + count := 0 + + _, err := ArrayEach(mock, func(value []byte, dataType ValueType, offset int, err *error) { + count++ + + if count == 2 { + *err = earlyTermError + } else if count > 2 { + t.Errorf("Should never go beyond second item") + } + }, "a") + + if err != earlyTermError { + t.Errorf("Expected error from early termination") + } +} + type keyValueEntry struct { key string value string