diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c9f5627..21d607c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,26 @@ +## 1.4.4 + +* New option `IgnoreUntaggedFields` to ignore decoding to any fields + without `mapstructure` (or the configured tag name) set [GH-277] +* New option `ErrorUnset` which makes it an error if any fields + in a target struct are not set by the decoding process. [GH-225] +* Decoding to slice from array no longer crashes [GH-265] +* Decode nested struct pointers to map [GH-271] +* Fix issue where `,squash` was ignored if `Squash` option was set. [GH-280] + +## 1.4.3 + +* Fix cases where `json.Number` didn't decode properly [GH-261] + +## 1.4.2 + +* Custom name matchers to support any sort of casing, formatting, etc. for + field names. [GH-250] +* Fix possible panic in ComposeDecodeHookFunc [GH-251] + ## 1.4.1 -* Fix regression where `*time.Time` value would be set to empty and not be sent +* Fix regression where `*time.Time` value would be set to empty and not be sent to decode hooks properly [GH-232] ## 1.4.0 diff --git a/decode_hooks.go b/decode_hooks.go index 28be794f..3a754ca7 100644 --- a/decode_hooks.go +++ b/decode_hooks.go @@ -62,7 +62,8 @@ func DecodeHookExec( func ComposeDecodeHookFunc(fs ...DecodeHookFunc) DecodeHookFunc { return func(f reflect.Value, t reflect.Value) (interface{}, error) { var err error - var data interface{} + data := f.Interface() + newFrom := f for _, f1 := range fs { data, err = DecodeHookExec(f1, newFrom, t) diff --git a/decode_hooks_test.go b/decode_hooks_test.go index 3f19c04e..bf029526 100644 --- a/decode_hooks_test.go +++ b/decode_hooks_test.go @@ -172,6 +172,38 @@ func TestOrComposeDecodeHookFunc_err(t *testing.T) { } } +func TestComposeDecodeHookFunc_safe_nofuncs(t *testing.T) { + f := ComposeDecodeHookFunc() + type myStruct2 struct { + MyInt int + } + + type myStruct1 struct { + Blah map[string]myStruct2 + } + + src := &myStruct1{Blah: map[string]myStruct2{ + "test": { + MyInt: 1, + }, + }} + + dst := &myStruct1{} + dConf := &DecoderConfig{ + Result: dst, + ErrorUnused: true, + DecodeHook: f, + } + d, err := NewDecoder(dConf) + if err != nil { + t.Fatal(err) + } + err = d.Decode(src) + if err != nil { + t.Fatal(err) + } +} + func TestStringToSliceHookFunc(t *testing.T) { f := StringToSliceHookFunc(",") diff --git a/mapstructure.go b/mapstructure.go index 3643901f..32c9fce7 100644 --- a/mapstructure.go +++ b/mapstructure.go @@ -122,7 +122,7 @@ // field value is zero and a numeric type, the field is empty, and it won't // be encoded into the destination type. // -// type Source { +// type Source struct { // Age int `mapstructure:",omitempty"` // } // @@ -192,7 +192,7 @@ type DecodeHookFuncType func(reflect.Type, reflect.Type, interface{}) (interface // source and target types. type DecodeHookFuncKind func(reflect.Kind, reflect.Kind, interface{}) (interface{}, error) -// DecodeHookFuncRaw is a DecodeHookFunc which has complete access to both the source and target +// DecodeHookFuncValue is a DecodeHookFunc which has complete access to both the source and target // values. type DecodeHookFuncValue func(from reflect.Value, to reflect.Value) (interface{}, error) @@ -215,6 +215,12 @@ type DecoderConfig struct { // (extra keys). ErrorUnused bool + // If ErrorUnset is true, then it is an error for there to exist + // fields in the result that were not set in the decoding process + // (extra fields). This only applies to decoding to a struct. This + // will affect all nested structs as well. + ErrorUnset bool + // ZeroFields, if set to true, will zero fields before writing them. // For example, a map will be emptied before decoded values are put in // it. If this is false, a map will be merged. @@ -258,6 +264,15 @@ type DecoderConfig struct { // The tag name that mapstructure reads for field names. This // defaults to "mapstructure" TagName string + + // IgnoreUntaggedFields ignores all struct fields without explicit + // TagName, comparable to `mapstructure:"-"` as default behaviour. + IgnoreUntaggedFields bool + + // MatchName is the function used to match the map key to the struct + // field name or tag. Defaults to `strings.EqualFold`. This can be used + // to implement case-sensitive tag values, support snake casing, etc. + MatchName func(mapKey, fieldName string) bool } // A Decoder takes a raw interface value and turns it into structured @@ -279,6 +294,11 @@ type Metadata struct { // Unused is a slice of keys that were found in the raw value but // weren't decoded since there was no matching field in the result interface Unused []string + + // Unset is a slice of field names that were found in the result interface + // but weren't set in the decoding process since there was no matching value + // in the input + Unset []string } // Decode takes an input structure and uses reflection to translate it to @@ -370,12 +390,20 @@ func NewDecoder(config *DecoderConfig) (*Decoder, error) { if config.Metadata.Unused == nil { config.Metadata.Unused = make([]string, 0) } + + if config.Metadata.Unset == nil { + config.Metadata.Unset = make([]string, 0) + } } if config.TagName == "" { config.TagName = "mapstructure" } + if config.MatchName == nil { + config.MatchName = strings.EqualFold + } + result := &Decoder{ config: config, } @@ -675,16 +703,12 @@ func (d *Decoder) decodeUint(name string, data interface{}, val reflect.Value) e } case dataType.PkgPath() == "encoding/json" && dataType.Name() == "Number": jn := data.(json.Number) - i, err := jn.Int64() + i, err := strconv.ParseUint(string(jn), 0, 64) if err != nil { return fmt.Errorf( "error decoding json.Number into %s: %s", name, err) } - if i < 0 && !d.config.WeaklyTypedInput { - return fmt.Errorf("cannot parse '%s', %d overflows uint", - name, i) - } - val.SetUint(uint64(i)) + val.SetUint(i) default: return fmt.Errorf( "'%s' expected type '%s', got unconvertible type '%s', value: '%v'", @@ -901,9 +925,15 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re tagValue := f.Tag.Get(d.config.TagName) keyName := f.Name + if tagValue == "" && d.config.IgnoreUntaggedFields { + continue + } + // If Squash is set in the config, we squash the field down. squash := d.config.Squash && v.Kind() == reflect.Struct && f.Anonymous + v = dereferencePtrToStructIfNeeded(v, d.config.TagName) + // Determine the name of the key in the map if index := strings.Index(tagValue, ","); index != -1 { if tagValue[:index] == "-" { @@ -915,7 +945,7 @@ func (d *Decoder) decodeMapFromStruct(name string, dataVal reflect.Value, val re } // If "squash" is specified in the tag, we squash the field down. - squash = !squash && strings.Index(tagValue[index+1:], "squash") != -1 + squash = squash || strings.Index(tagValue[index+1:], "squash") != -1 if squash { // When squashing, the embedded type can be a pointer to a struct. if v.Kind() == reflect.Ptr && v.Elem().Kind() == reflect.Struct { @@ -1083,7 +1113,7 @@ func (d *Decoder) decodeSlice(name string, data interface{}, val reflect.Value) } // If the input value is nil, then don't allocate since empty != nil - if dataVal.IsNil() { + if dataValKind != reflect.Array && dataVal.IsNil() { return nil } @@ -1245,6 +1275,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e dataValKeysUnused[dataValKey.Interface()] = struct{}{} } + targetValKeysUnused := make(map[interface{}]struct{}) errors := make([]string, 0) // This slice will keep track of all the structs we'll be decoding. @@ -1340,7 +1371,7 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e continue } - if strings.EqualFold(mK, fieldName) { + if d.config.MatchName(mK, fieldName) { rawMapKey = dataValKey rawMapVal = dataVal.MapIndex(dataValKey) break @@ -1349,7 +1380,8 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e if !rawMapVal.IsValid() { // There was no matching key in the map for the value in - // the struct. Just ignore. + // the struct. Remember it for potential errors and metadata. + targetValKeysUnused[fieldName] = struct{}{} continue } } @@ -1409,6 +1441,17 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e errors = appendErrors(errors, err) } + if d.config.ErrorUnset && len(targetValKeysUnused) > 0 { + keys := make([]string, 0, len(targetValKeysUnused)) + for rawKey := range targetValKeysUnused { + keys = append(keys, rawKey.(string)) + } + sort.Strings(keys) + + err := fmt.Errorf("'%s' has unset fields: %s", name, strings.Join(keys, ", ")) + errors = appendErrors(errors, err) + } + if len(errors) > 0 { return &Error{errors} } @@ -1423,6 +1466,14 @@ func (d *Decoder) decodeStructFromMap(name string, dataVal, val reflect.Value) e d.config.Metadata.Unused = append(d.config.Metadata.Unused, key) } + for rawKey := range targetValKeysUnused { + key := rawKey.(string) + if name != "" { + key = name + "." + key + } + + d.config.Metadata.Unset = append(d.config.Metadata.Unset, key) + } } return nil @@ -1460,3 +1511,28 @@ func getKind(val reflect.Value) reflect.Kind { return kind } } + +func isStructTypeConvertibleToMap(typ reflect.Type, checkMapstructureTags bool, tagName string) bool { + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if f.PkgPath == "" && !checkMapstructureTags { // check for unexported fields + return true + } + if checkMapstructureTags && f.Tag.Get(tagName) != "" { // check for mapstructure tags inside + return true + } + } + return false +} + +func dereferencePtrToStructIfNeeded(v reflect.Value, tagName string) reflect.Value { + if v.Kind() != reflect.Ptr || v.Elem().Kind() != reflect.Struct { + return v + } + deref := v.Elem() + derefT := deref.Type() + if isStructTypeConvertibleToMap(derefT, true, tagName) { + return deref + } + return v +} diff --git a/mapstructure_bugs_test.go b/mapstructure_bugs_test.go index bcfc5054..f0637606 100644 --- a/mapstructure_bugs_test.go +++ b/mapstructure_bugs_test.go @@ -479,7 +479,7 @@ func TestDecodeBadDataTypeInSlice(t *testing.T) { // #202 Ensure that intermediate maps in the struct -> struct decode process are settable // and not just the elements within them. -func TestDecodeIntermeidateMapsSettable(t *testing.T) { +func TestDecodeIntermediateMapsSettable(t *testing.T) { type Timestamp struct { Seconds int64 Nanos int32 diff --git a/mapstructure_test.go b/mapstructure_test.go index 53b2d0a4..bef5f85d 100644 --- a/mapstructure_test.go +++ b/mapstructure_test.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "testing" + "time" ) type Basic struct { @@ -24,6 +25,7 @@ type Basic struct { Vdata interface{} VjsonInt int VjsonUint uint + VjsonUint64 uint64 VjsonFloat float64 VjsonNumber json.Number } @@ -66,6 +68,20 @@ type EmbeddedPointerSquash struct { Vunique string } +type BasicMapStructure struct { + Vunique string `mapstructure:"vunique"` + Vtime *time.Time `mapstructure:"time"` +} + +type NestedPointerWithMapstructure struct { + Vbar *BasicMapStructure `mapstructure:"vbar"` +} + +type EmbeddedPointerSquashWithNestedMapstructure struct { + *NestedPointerWithMapstructure `mapstructure:",squash"` + Vunique string +} + type EmbeddedAndNamed struct { Basic Named Basic @@ -224,6 +240,7 @@ func TestBasicTypes(t *testing.T) { "vdata": 42, "vjsonInt": json.Number("1234"), "vjsonUint": json.Number("1234"), + "vjsonUint64": json.Number("9223372036854775809"), // 2^63 + 1 "vjsonFloat": json.Number("1234.5"), "vjsonNumber": json.Number("1234.5"), } @@ -287,6 +304,10 @@ func TestBasicTypes(t *testing.T) { t.Errorf("vjsonuint value should be 1234: %#v", result.VjsonUint) } + if result.VjsonUint64 != 9223372036854775809 { + t.Errorf("vjsonuint64 value should be 9223372036854775809: %#v", result.VjsonUint64) + } + if result.VjsonFloat != 1234.5 { t.Errorf("vjsonfloat value should be 1234.5: %#v", result.VjsonFloat) } @@ -550,6 +571,21 @@ func TestDecode_EmbeddedArray(t *testing.T) { } } +func TestDecode_decodeSliceWithArray(t *testing.T) { + t.Parallel() + + var result []int + input := [1]int{1} + expected := []int{1} + if err := Decode(input, &result); err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + if !reflect.DeepEqual(expected, result) { + t.Errorf("wanted %+v, got %+v", expected, result) + } +} + func TestDecode_EmbeddedNoSquash(t *testing.T) { t.Parallel() @@ -710,6 +746,74 @@ func TestDecode_EmbeddedPointerSquash_FromMapToStruct(t *testing.T) { } } +func TestDecode_EmbeddedPointerSquashWithNestedMapstructure_FromStructToMap(t *testing.T) { + t.Parallel() + + vTime := time.Now() + + input := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{ + Vbar: &BasicMapStructure{ + Vunique: "bar", + Vtime: &vTime, + }, + }, + Vunique: "foo", + } + + var result map[string]interface{} + err := Decode(input, &result) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + expected := map[string]interface{}{ + "vbar": map[string]interface{}{ + "vunique": "bar", + "time": &vTime, + }, + "Vunique": "foo", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("result should be %#v: got %#v", expected, result) + } +} + +func TestDecode_EmbeddedPointerSquashWithNestedMapstructure_FromMapToStruct(t *testing.T) { + t.Parallel() + + vTime := time.Now() + + input := map[string]interface{}{ + "vbar": map[string]interface{}{ + "vunique": "bar", + "time": &vTime, + }, + "Vunique": "foo", + } + + result := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{}, + } + err := Decode(input, &result) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + expected := EmbeddedPointerSquashWithNestedMapstructure{ + NestedPointerWithMapstructure: &NestedPointerWithMapstructure{ + Vbar: &BasicMapStructure{ + Vunique: "bar", + Vtime: &vTime, + }, + }, + Vunique: "foo", + } + + if !reflect.DeepEqual(result, expected) { + t.Errorf("result should be %#v: got %#v", expected, result) + } +} + func TestDecode_EmbeddedSquashConfig(t *testing.T) { t.Parallel() @@ -806,6 +910,53 @@ func TestDecodeFrom_EmbeddedSquashConfig(t *testing.T) { } } +func TestDecodeFrom_EmbeddedSquashConfig_WithTags(t *testing.T) { + t.Parallel() + + var v interface{} + var ok bool + + input := EmbeddedSquash{ + Basic: Basic{ + Vstring: "foo", + }, + Vunique: "bar", + } + + result := map[string]interface{}{} + config := &DecoderConfig{ + Squash: true, + Result: &result, + } + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("got an err: %s", err.Error()) + } + + if _, ok = result["Basic"]; ok { + t.Error("basic should not be present in map") + } + + v, ok = result["Vstring"] + if !ok { + t.Error("vstring should be present in map") + } else if !reflect.DeepEqual(v, "foo") { + t.Errorf("vstring value should be 'foo': %#v", v) + } + + v, ok = result["Vunique"] + if !ok { + t.Error("vunique should be present in map") + } else if !reflect.DeepEqual(v, "bar") { + t.Errorf("vunique value should be 'bar': %#v", v) + } +} + func TestDecode_SquashOnNonStructType(t *testing.T) { t.Parallel() @@ -1225,6 +1376,30 @@ func TestDecoder_ErrorUnused_NotSetable(t *testing.T) { t.Fatal("expected error") } } +func TestDecoder_ErrorUnset(t *testing.T) { + t.Parallel() + + input := map[string]interface{}{ + "vstring": "hello", + "foo": "bar", + } + + var result Basic + config := &DecoderConfig{ + ErrorUnset: true, + Result: &result, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err == nil { + t.Fatal("expected error") + } +} func TestMap(t *testing.T) { t.Parallel() @@ -1721,6 +1896,7 @@ func TestDecodeTable(t *testing.T) { "Vdata": []byte("data"), "VjsonInt": 0, "VjsonUint": uint(0), + "VjsonUint64": uint64(0), "VjsonFloat": 0.0, "VjsonNumber": json.Number(""), }, @@ -1762,6 +1938,7 @@ func TestDecodeTable(t *testing.T) { "Vdata": []byte("data"), "VjsonInt": 0, "VjsonUint": uint(0), + "VjsonUint64": uint64(0), "VjsonFloat": 0.0, "VjsonNumber": json.Number(""), }, @@ -2192,6 +2369,14 @@ func TestMetadata(t *testing.T) { if !reflect.DeepEqual(md.Unused, expectedUnused) { t.Fatalf("bad unused: %#v", md.Unused) } + + expectedUnset := []string{ + "Vbar.Vbool", "Vbar.Vdata", "Vbar.Vextra", "Vbar.Vfloat", "Vbar.Vint", + "Vbar.VjsonFloat", "Vbar.VjsonInt", "Vbar.VjsonNumber"} + sort.Strings(md.Unset) + if !reflect.DeepEqual(md.Unset, expectedUnset) { + t.Fatalf("bad unset: %#v", md.Unset) + } } func TestMetadata_Embedded(t *testing.T) { @@ -2431,6 +2616,89 @@ func TestDecode_mapToStruct(t *testing.T) { } } +func TestDecoder_MatchName(t *testing.T) { + t.Parallel() + + type Target struct { + FirstMatch string `mapstructure:"first_match"` + SecondMatch string + NoMatch string `mapstructure:"no_match"` + } + + input := map[string]interface{}{ + "first_match": "foo", + "SecondMatch": "bar", + "NO_MATCH": "baz", + } + + expected := Target{ + FirstMatch: "foo", + SecondMatch: "bar", + } + + var actual Target + config := &DecoderConfig{ + Result: &actual, + MatchName: func(mapKey, fieldName string) bool { + return mapKey == fieldName + }, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("err: %s", err) + } + + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Decode() expected: %#v, got: %#v", expected, actual) + } +} + +func TestDecoder_IgnoreUntaggedFields(t *testing.T) { + type Input struct { + UntaggedNumber int + TaggedNumber int `mapstructure:"tagged_number"` + UntaggedString string + TaggedString string `mapstructure:"tagged_string"` + } + input := &Input{ + UntaggedNumber: 31, + TaggedNumber: 42, + UntaggedString: "hidden", + TaggedString: "visible", + } + + actual := make(map[string]interface{}) + config := &DecoderConfig{ + Result: &actual, + IgnoreUntaggedFields: true, + } + + decoder, err := NewDecoder(config) + if err != nil { + t.Fatalf("err: %s", err) + } + + err = decoder.Decode(input) + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := map[string]interface{}{ + "tagged_number": 42, + "tagged_string": "visible", + } + + if !reflect.DeepEqual(expected, actual) { + t.Fatalf("Decode() expected: %#v\ngot: %#v", expected, actual) + } +} + func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) { var result Slice err := Decode(input, &result)