diff --git a/config.go b/config.go index 2adcdc3b..8ec637d5 100644 --- a/config.go +++ b/config.go @@ -20,6 +20,7 @@ type Config struct { SortMapKeys bool UseNumber bool DisallowUnknownFields bool + DisallowDuplicateFields bool TagKey string OnlyTaggedField bool ValidateJsonRawMessage bool @@ -72,6 +73,7 @@ type frozenConfig struct { objectFieldMustBeSimpleString bool onlyTaggedField bool disallowUnknownFields bool + disallowDuplicateFields bool decoderCache *concurrent.Map encoderCache *concurrent.Map encoderExtension Extension @@ -133,6 +135,7 @@ func (cfg Config) Froze() API { objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString, onlyTaggedField: cfg.OnlyTaggedField, disallowUnknownFields: cfg.DisallowUnknownFields, + disallowDuplicateFields: cfg.DisallowDuplicateFields, caseSensitive: cfg.CaseSensitive, } api.streamPool = &sync.Pool{ diff --git a/example_test.go b/example_test.go index 1516b77b..7abdbcf4 100644 --- a/example_test.go +++ b/example_test.go @@ -4,6 +4,9 @@ import ( "fmt" "os" "strings" + "testing" + + "github.com/stretchr/testify/assert" ) func ExampleMarshal() { @@ -119,3 +122,129 @@ func (m *MyKey) UnmarshalText(text []byte) error { *m = MyKey(text[:3]) return nil } + +type Target struct { + FieldA string `json:"fieldA"` +} + +func Example_duplicateFieldsCaseSensitive() { + api := Config{ + CaseSensitive: true, + DisallowDuplicateFields: true, + }.Froze() + + t := &Target{} + err := api.Unmarshal([]byte(`{"fieldA": "value", "fielda": "val2"}`), t) + fmt.Printf("Case-sensitiveness means no duplicates: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fieldA": "value", "fieldA": "val2"}`), t) + fmt.Printf("Got duplicates in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fielda": "value", "fielda": "val2"}`), t) + fmt.Printf("Got duplicates not in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + // Output: + // Case-sensitiveness means no duplicates: 'fieldA' = "value", err = + // Got duplicates in struct field: 'fieldA' = "value", err = jsoniter.Target.ReadObject: found duplicate field: fieldA, error found in #10 byte of ...|, "fieldA": "val2"}|..., bigger context ...|{"fieldA": "value", "fieldA": "val2"}|... + // Got duplicates not in struct field: 'fieldA' = "", err = jsoniter.Target.ReadObject: found duplicate field: fielda, error found in #10 byte of ...|, "fielda": "val2"}|..., bigger context ...|{"fielda": "value", "fielda": "val2"}|... +} + +func Example_noDuplicateFieldsCaseSensitive() { + api := Config{ + CaseSensitive: true, + DisallowDuplicateFields: false, + }.Froze() + + t := &Target{} + err := api.Unmarshal([]byte(`{"fieldA": "value", "fielda": "val2"}`), t) + fmt.Printf("Case-sensitiveness means no duplicates: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fieldA": "value", "fieldA": "val2"}`), t) + fmt.Printf("Got duplicates in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fielda": "value", "fielda": "val2"}`), t) + fmt.Printf("Got duplicates not in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + // Output: + // Case-sensitiveness means no duplicates: 'fieldA' = "value", err = + // Got duplicates in struct field: 'fieldA' = "val2", err = + // Got duplicates not in struct field: 'fieldA' = "", err = +} + +func Example_duplicateFieldsInCaseSensitive() { + api := Config{ + CaseSensitive: false, + DisallowDuplicateFields: true, + }.Froze() + + t := &Target{} + err := api.Unmarshal([]byte(`{"fieldA": "value", "fielda": "val2"}`), t) + fmt.Printf("In-case-sensitive duplicates: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fieldA": "value", "fieldA": "val2"}`), t) + fmt.Printf("Got duplicates in exact struct field match: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fielda": "value", "fielda": "val2"}`), t) + fmt.Printf("Got duplicates not in notexact struct field match: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + // Output: + // In-case-sensitive duplicates: 'fieldA' = "value", err = jsoniter.Target.ReadObject: found duplicate field: fielda, error found in #10 byte of ...|, "fielda": "val2"}|..., bigger context ...|{"fieldA": "value", "fielda": "val2"}|... + // Got duplicates in exact struct field match: 'fieldA' = "value", err = jsoniter.Target.ReadObject: found duplicate field: fieldA, error found in #10 byte of ...|, "fieldA": "val2"}|..., bigger context ...|{"fieldA": "value", "fieldA": "val2"}|... + // Got duplicates not in notexact struct field match: 'fieldA' = "value", err = jsoniter.Target.ReadObject: found duplicate field: fielda, error found in #10 byte of ...|, "fielda": "val2"}|..., bigger context ...|{"fielda": "value", "fielda": "val2"}|... +} + +func Example_noDuplicateFieldsInCaseSensitive() { + api := Config{ + CaseSensitive: false, + DisallowDuplicateFields: false, + }.Froze() + + t := &Target{} + err := api.Unmarshal([]byte(`{"fieldA": "value", "fielda": "val2"}`), t) + fmt.Printf("Case-sensitiveness means no duplicates: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fieldA": "value", "fieldA": "val2"}`), t) + fmt.Printf("Got duplicates in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = api.Unmarshal([]byte(`{"fielda": "value", "fielda": "val2"}`), t) + fmt.Printf("Got duplicates not in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + // Output: + // Case-sensitiveness means no duplicates: 'fieldA' = "val2", err = + // Got duplicates in struct field: 'fieldA' = "val2", err = + // Got duplicates not in struct field: 'fieldA' = "val2", err = +} + +func TestEncoder(t *testing.T) { + api := Config{ + CaseSensitive: true, + DisallowDuplicateFields: true, + }.Froze() + + type target2 struct { + B Target `json:"b"` + A Target `json:"a"` + } + + data := `{"a": {"fieldA": "bla"}, "b": {"fieldA": "bar"}}` + data += data + + d := api.NewDecoder(strings.NewReader(data)) + obj := &target2{} + assert.Nil(t, d.Decode(obj)) + assert.Equal(t, "bla", obj.A.FieldA) + assert.Equal(t, "bar", obj.B.FieldA) + + obj = &target2{} + assert.Nil(t, d.Decode(obj)) + assert.Equal(t, "bla", obj.A.FieldA) + assert.Equal(t, "bar", obj.B.FieldA) +} diff --git a/iter_object.go b/iter_object.go index 58ee89c8..80b460f7 100644 --- a/iter_object.go +++ b/iter_object.go @@ -2,7 +2,6 @@ package jsoniter import ( "fmt" - "strings" ) // ReadObject read one field from object. @@ -95,10 +94,7 @@ func (iter *Iterator) readFieldHash() int64 { } } -func calcHash(str string, caseSensitive bool) int64 { - if !caseSensitive { - str = strings.ToLower(str) - } +func calcHash(str string) int64 { hash := int64(0x811c9dc5) for _, b := range []byte(str) { hash ^= int64(b) diff --git a/reflect_struct_decoder.go b/reflect_struct_decoder.go index 92ae912d..e8f7b7f5 100644 --- a/reflect_struct_decoder.go +++ b/reflect_struct_decoder.go @@ -45,8 +45,8 @@ func decoderOfStruct(ctx *ctx, typ reflect2.Type) ValDecoder { } func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structFieldDecoder) ValDecoder { - if ctx.disallowUnknownFields { - return &generalStructDecoder{typ: typ, fields: fields, disallowUnknownFields: true} + if ctx.disallowUnknownFields || ctx.disallowDuplicateFields || !ctx.caseSensitive() { + return &generalStructDecoder{typ, fields} } knownHash := map[int64]struct{}{ 0: {}, @@ -57,10 +57,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF return &skipObjectDecoder{typ} case 1: for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} return &oneFieldStructDecoder{typ, fieldHash, fieldDecoder} @@ -71,10 +71,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder1 *structFieldDecoder var fieldDecoder2 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldHash1 == 0 { @@ -94,10 +94,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder2 *structFieldDecoder var fieldDecoder3 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -125,10 +125,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder3 *structFieldDecoder var fieldDecoder4 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -162,10 +162,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder4 *structFieldDecoder var fieldDecoder5 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -205,10 +205,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder5 *structFieldDecoder var fieldDecoder6 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -254,10 +254,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder6 *structFieldDecoder var fieldDecoder7 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -309,10 +309,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder7 *structFieldDecoder var fieldDecoder8 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -370,10 +370,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder8 *structFieldDecoder var fieldDecoder9 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -437,10 +437,10 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF var fieldDecoder9 *structFieldDecoder var fieldDecoder10 *structFieldDecoder for fieldName, fieldDecoder := range fields { - fieldHash := calcHash(fieldName, ctx.caseSensitive()) + fieldHash := calcHash(fieldName) _, known := knownHash[fieldHash] if known { - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } knownHash[fieldHash] = struct{}{} if fieldName1 == 0 { @@ -487,13 +487,12 @@ func createStructDecoder(ctx *ctx, typ reflect2.Type, fields map[string]*structF fieldName9, fieldDecoder9, fieldName10, fieldDecoder10} } - return &generalStructDecoder{typ, fields, false} + return &generalStructDecoder{typ, fields} } type generalStructDecoder struct { - typ reflect2.Type - fields map[string]*structFieldDecoder - disallowUnknownFields bool + typ reflect2.Type + fields map[string]*structFieldDecoder } func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { @@ -503,9 +502,13 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) if !iter.incrementDepth() { return } + var seenFields map[string]struct{} + if iter.cfg.disallowDuplicateFields { + seenFields = make(map[string]struct{}, len(decoder.fields)) + } var c byte for c = ','; c == ','; c = iter.nextToken() { - decoder.decodeOneField(ptr, iter) + decoder.decodeOneField(ptr, iter, seenFields) } if iter.Error != nil && iter.Error != io.EOF && len(decoder.typ.Type1().Name()) != 0 { iter.Error = fmt.Errorf("%v.%s", decoder.typ, iter.Error.Error()) @@ -516,28 +519,42 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) iter.decrementDepth() } -func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator) { +func (decoder *generalStructDecoder) decodeOneField(ptr unsafe.Pointer, iter *Iterator, seenFields map[string]struct{}) { var field string - var fieldDecoder *structFieldDecoder if iter.cfg.objectFieldMustBeSimpleString { fieldBytes := iter.ReadStringAsSlice() field = *(*string)(unsafe.Pointer(&fieldBytes)) - fieldDecoder = decoder.fields[field] - if fieldDecoder == nil && !iter.cfg.caseSensitive { - fieldDecoder = decoder.fields[strings.ToLower(field)] - } } else { field = iter.ReadString() - fieldDecoder = decoder.fields[field] - if fieldDecoder == nil && !iter.cfg.caseSensitive { - fieldDecoder = decoder.fields[strings.ToLower(field)] + } + + fieldDecoder := decoder.fields[field] + if fieldDecoder == nil && !iter.cfg.caseSensitive { + fieldDecoder = decoder.fields[strings.ToLower(field)] + } + + isDuplicate := false + if iter.cfg.disallowDuplicateFields { + seenField := field + if !iter.cfg.caseSensitive { + seenField = strings.ToLower(seenField) } + _, isDuplicate = seenFields[seenField] + // Always register here so that also unknown fields' duplicates are + // registered. + seenFields[seenField] = struct{}{} } - if fieldDecoder == nil { - if decoder.disallowUnknownFields { + + if fieldDecoder == nil || isDuplicate { + if iter.cfg.disallowUnknownFields { msg := "found unknown field: " + field iter.ReportError("ReadObject", msg) } + if isDuplicate { + msg := "found duplicate field: " + field + iter.ReportError("ReadObject", msg) + } + // Skip field c := iter.nextToken() if c != ':' { iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))