15 support capturing nested unknown fields (#17)

* implement JSONDataHandler support

* implement JSONDataHandler support

* add github stats badges

README.md

@@ -6,10 +6,14 @@
 [![Run Tests](https://img.shields.io/github/workflow/status/perimeterx/marshmallow/Go?label=Run%20Tests&logo=github)](https://github.com/PerimeterX/marshmallow/actions/workflows/go.yml?query=branch%3Amain)
 [![Dependency Review](https://img.shields.io/github/workflow/status/perimeterx/marshmallow/Dependency%20Review?label=Dependency%20Review&logo=github)](https://github.com/PerimeterX/marshmallow/actions/workflows/dependency-review.yml?query=branch%3Amain)
 [![Go Report Card](https://goreportcard.com/badge/github.com/perimeterx/marshmallow)](https://goreportcard.com/report/github.com/perimeterx/marshmallow)
-![Manual Code Coverage](https://img.shields.io/badge/coverage-91.3%25-green)
+![Manual Code Coverage](https://img.shields.io/badge/coverage-91.9%25-green)
 [![Go Reference](https://pkg.go.dev/badge/github.com/perimeterx/marshmallow.svg)](https://pkg.go.dev/github.com/perimeterx/marshmallow)
 [![Licence](https://img.shields.io/github/license/perimeterx/marshmallow)](LICENSE)
 [![Latest Release](https://img.shields.io/github/v/release/perimeterx/marshmallow)](https://github.com/PerimeterX/marshmallow/releases)
+![Top Languages](https://img.shields.io/github/languages/top/perimeterx/marshmallow)
+[![Issues](https://img.shields.io/github/issues-closed/perimeterx/marshmallow?color=%238250df&logo=github)](https://github.com/PerimeterX/marshmallow/issues)
+[![Pull Requests](https://img.shields.io/github/issues-pr-closed-raw/perimeterx/marshmallow?color=%238250df&label=merged%20pull%20requests&logo=github)](https://github.com/PerimeterX/marshmallow/pulls)
+[![Commits](https://img.shields.io/github/last-commit/perimeterx/marshmallow)](https://github.com/PerimeterX/marshmallow/commits/main)
 
 <img align="right" width="200" alt="marshmallow-gopher" src="https://raw.githubusercontent.com/PerimeterX/marshmallow/assets/sticker7.png">
 
@@ -168,11 +172,16 @@ and
 Each of them can operate in three possible [modes](https://github.com/PerimeterX/marshmallow/blob/0e0218ab860be8a4b5f57f5ff239f281c250c5da/options.go#L30),
 and allow setting [skipPopulateStruct](https://github.com/PerimeterX/marshmallow/blob/0e0218ab860be8a4b5f57f5ff239f281c250c5da/options.go#L41) mode.
 
+In order to capture unknown nested fields, structs must implement [JSONDataHandler](https://github.com/PerimeterX/marshmallow/blob/2d254bf2ed5f9b02cafb8ba6eaa726cba38bc92b/options.go#L65).
+More info [here](https://github.com/PerimeterX/marshmallow/issues/15). 
+
 Marshmallow also supports caching of refection information using 
 [EnableCache](https://github.com/PerimeterX/marshmallow/blob/d3500aa5b0f330942b178b155da933c035dd3906/cache.go#L40)
 and
 [EnableCustomCache](https://github.com/PerimeterX/marshmallow/blob/d3500aa5b0f330942b178b155da933c035dd3906/cache.go#L35).
 
+**Examples can be found [here](example_test.go)**
+
 # Marshmallow Logo
 
 Marshmallow logo and assets by [Adva Rom](https://www.linkedin.com/in/adva-rom-7a6738127/) are licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>.<br />

example_test.go

@@ -5,7 +5,7 @@ import (
 	"github.com/perimeterx/marshmallow"
 )
 
-type exampleStruct struct {
+type flatStruct struct {
 	Foo string `json:"foo"`
 	Boo []int  `json:"boo"`
 }
@@ -16,40 +16,40 @@ func ExampleUnmarshal() {
 
 	// unmarshal with mode marshmallow.ModeFailOnFirstError and valid value
 	// this will finish unmarshalling and return a nil err
-	v := exampleStruct{}
+	v := flatStruct{}
 	result, err := marshmallow.Unmarshal([]byte(`{"foo":"bar","boo":[1,2,3]}`), &v)
 	fmt.Printf("ModeFailOnFirstError and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOnFirstError and invalid value
 	// this will return nil result and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.Unmarshal([]byte(`{"foo":2,"boo":[1,2,3]}`), &v)
 	fmt.Printf("ModeFailOnFirstError and invalid value: result=%+v, err=%T\n", result, err)
 
 	// unmarshal with mode marshmallow.ModeAllowMultipleErrors and valid value
 	// this will finish unmarshalling and return a nil err
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.Unmarshal([]byte(`{"foo":"bar","boo":[1,2,3]}`), &v,
 		marshmallow.WithMode(marshmallow.ModeAllowMultipleErrors))
 	fmt.Printf("ModeAllowMultipleErrors and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeAllowMultipleErrors and invalid value
 	// this will return a partially populated result and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.Unmarshal([]byte(`{"foo":2,"boo":[1,2,3]}`), &v,
 		marshmallow.WithMode(marshmallow.ModeAllowMultipleErrors))
 	fmt.Printf("ModeAllowMultipleErrors and invalid value: result=%+v, err=%T\n", result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOverToOriginalValue and valid value
 	// this will finish unmarshalling and return a nil err
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.Unmarshal([]byte(`{"foo":"bar","boo":[1,2,3]}`), &v,
 		marshmallow.WithMode(marshmallow.ModeFailOverToOriginalValue))
 	fmt.Printf("ModeFailOverToOriginalValue and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOverToOriginalValue and invalid value
 	// this will return a fully unmarshalled result, failing to the original invalid values, and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.Unmarshal([]byte(`{"foo":2,"boo":[1,2,3]}`), &v,
 		marshmallow.WithMode(marshmallow.ModeFailOverToOriginalValue))
 	fmt.Printf("ModeFailOverToOriginalValue and invalid value: result=%+v, err=%T\n", result, err)
@@ -68,45 +68,45 @@ func ExampleUnmarshalFromJSONMap() {
 
 	// unmarshal with mode marshmallow.ModeFailOnFirstError and valid value
 	// this will finish unmarshalling and return a nil err
-	v := exampleStruct{}
+	v := flatStruct{}
 	result, err := marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": "bar", "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v)
 	fmt.Printf("ModeFailOnFirstError and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOnFirstError and invalid value
 	// this will return nil result and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": float64(2), "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v)
 	fmt.Printf("ModeFailOnFirstError and invalid value: result=%+v, err=%T\n", result, err)
 
 	// unmarshal with mode marshmallow.ModeAllowMultipleErrors and valid value
 	// this will finish unmarshalling and return a nil err
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": "bar", "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v,
 		marshmallow.WithMode(marshmallow.ModeAllowMultipleErrors))
 	fmt.Printf("ModeAllowMultipleErrors and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeAllowMultipleErrors and invalid value
 	// this will return a partially populated result and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": float64(2), "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v,
 		marshmallow.WithMode(marshmallow.ModeAllowMultipleErrors))
 	fmt.Printf("ModeAllowMultipleErrors and invalid value: result=%+v, err=%T\n", result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOverToOriginalValue and valid value
 	// this will finish unmarshalling and return a nil err
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": "bar", "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v,
 		marshmallow.WithMode(marshmallow.ModeFailOverToOriginalValue))
 	fmt.Printf("ModeFailOverToOriginalValue and valid value: v=%+v, result=%+v, err=%T\n", v, result, err)
 
 	// unmarshal with mode marshmallow.ModeFailOverToOriginalValue and invalid value
 	// this will return a fully unmarshalled result, failing to the original invalid values, and an error
-	v = exampleStruct{}
+	v = flatStruct{}
 	result, err = marshmallow.UnmarshalFromJSONMap(
 		map[string]interface{}{"foo": float64(2), "boo": []interface{}{float64(1), float64(2), float64(3)}}, &v,
 		marshmallow.WithMode(marshmallow.ModeFailOverToOriginalValue))
@@ -119,3 +119,29 @@ func ExampleUnmarshalFromJSONMap() {
 	// ModeFailOverToOriginalValue and valid value: v={Foo:bar Boo:[1 2 3]}, result=map[boo:[1 2 3] foo:bar], err=<nil>
 	// ModeFailOverToOriginalValue and invalid value: result=map[boo:[1 2 3] foo:2], err=*marshmallow.MultipleError
 }
+
+type parentStruct struct {
+	Known  string      `json:"known"`
+	Nested childStruct `json:"nested"`
+}
+
+type childStruct struct {
+	Known string `json:"known"`
+
+	Data map[string]interface{} `json:"-"`
+}
+
+func (c *childStruct) HandleJSONData(data map[string]interface{}) {
+	c.Data = data
+}
+
+func ExampleJSONDataHandler() {
+	data := []byte(`{"known": "foo","unknown": "boo","nested": {"known": "goo","unknown": "doo"}}`)
+	p := &parentStruct{}
+	_, err := marshmallow.Unmarshal(data, p)
+	fmt.Printf("err: %v\n", err)
+	fmt.Printf("nested data: %+v\n", p.Nested.Data)
+	// Output:
+	// err: <nil>
+	// nested data: map[known:goo unknown:doo]
+}

options.go

@@ -58,3 +58,10 @@ func buildUnmarshalOptions(options []UnmarshalOption) *unmarshalOptions {
 	}
 	return result
 }
+
+// JSONDataHandler allow types to handle JSON data as maps.
+// Types should implement this interface if they wish to act on the map representation of parsed JSON input.
+// This is mainly used to allow nested objects to capture unknown fields and leverage marshmallow's abilities.
+type JSONDataHandler interface {
+	HandleJSONData(data map[string]interface{})
+}

unmarshal.go

@@ -304,7 +304,16 @@ func (d *decoder) buildStruct(structType reflect.Type) (interface{}, bool) {
 		return d.lexer.Interface(), false
 	}
 	value := reflect.New(structType).Interface()
-	return d.populateStruct(value, nil)
+	handler, ok := value.(JSONDataHandler)
+	if !ok {
+		return d.populateStruct(value, nil)
+	}
+	data := make(map[string]interface{})
+	result, valid := d.populateStruct(value, data)
+	if valid {
+		handler.HandleJSONData(data)
+	}
+	return result, valid
 }
 
 func (d *decoder) valueFromCustomUnmarshaler(unmarshaler json.Unmarshaler) {

unmarshal_from_json_map.go

@@ -260,7 +260,16 @@ func (m *mapDecoder) buildStruct(path []string, v interface{}, structType reflec
 		return v, false
 	}
 	value := reflect.New(structType).Interface()
-	return m.populateStruct(path, mp, value, nil)
+	handler, ok := value.(JSONDataHandler)
+	if !ok {
+		return m.populateStruct(path, mp, value, nil)
+	}
+	data := make(map[string]interface{})
+	result, valid := m.populateStruct(path, mp, value, data)
+	if valid {
+		handler.HandleJSONData(data)
+	}
+	return result, valid
 }
 
 func (m *mapDecoder) valueFromCustomUnmarshaler(data interface{}, unmarshaler UnmarshalerFromJSONMap) {

unmarshal_from_json_map_test.go

@@ -2260,7 +2260,7 @@ func TestUnmarshalFromJSONMapSpecialInput(t *testing.T) {
 
 func TestUnmarshalFromJSONMapEmbedding(t *testing.T) {
 	t.Run("test_embedded_values", func(t *testing.T) {
-		p := parent{}
+		p := embeddingParent{}
 		result, err := UnmarshalFromJSONMap(map[string]interface{}{"field": "value"}, &p)
 		if err != nil {
 			t.Errorf("unexpected error %v", err)
@@ -2273,3 +2273,31 @@ func TestUnmarshalFromJSONMapEmbedding(t *testing.T) {
 		}
 	})
 }
+
+func TestUnmarshalFromJSONMapJSONDataHandler(t *testing.T) {
+	t.Run("test_JSONDataHandler", func(t *testing.T) {
+		data := map[string]interface{}{
+			"known":   "foo",
+			"unknown": "boo",
+			"nested": map[string]interface{}{
+				"known":   "goo",
+				"unknown": "doo",
+			},
+		}
+		p := &handleJSONDataParent{}
+		result, err := UnmarshalFromJSONMap(data, p)
+		if err != nil {
+			t.Errorf("unexpected error %v", err)
+		}
+		_, ok := result["nested"].(handleJSONDataChild)
+		if !ok {
+			t.Error("invalid map value")
+		}
+		if p.Nested.Data == nil {
+			t.Error("HandleJSONData not called")
+		}
+		if len(p.Nested.Data) != 2 || p.Nested.Data["known"] != "goo" || p.Nested.Data["unknown"] != "doo" {
+			t.Error("invalid JSON data")
+		}
+	})
+}

unmarshal_test.go

@@ -2281,17 +2281,17 @@ func TestUnmarshalSpecialInput(t *testing.T) {
 	}
 }
 
-type parent struct {
-	child
+type embeddingParent struct {
+	embeddingChild
 }
 
-type child struct {
+type embeddingChild struct {
 	Field string `json:"field"`
 }
 
 func TestEmbedding(t *testing.T) {
 	t.Run("test_embedded_values", func(t *testing.T) {
-		p := parent{}
+		p := embeddingParent{}
 		result, err := Unmarshal([]byte(`{"field":"value"}`), &p)
 		if err != nil {
 			t.Errorf("unexpected error %v", err)
@@ -2305,6 +2305,42 @@ func TestEmbedding(t *testing.T) {
 	})
 }
 
+type handleJSONDataParent struct {
+	Known  string              `json:"known"`
+	Nested handleJSONDataChild `json:"nested"`
+}
+
+type handleJSONDataChild struct {
+	Known string `json:"known"`
+
+	Data map[string]interface{} `json:"-"`
+}
+
+func (c *handleJSONDataChild) HandleJSONData(data map[string]interface{}) {
+	c.Data = data
+}
+
+func TestJSONDataHandler(t *testing.T) {
+	t.Run("test_JSONDataHandler", func(t *testing.T) {
+		data := []byte(`{"known": "foo","unknown": "boo","nested": {"known": "goo","unknown": "doo"}}`)
+		p := &handleJSONDataParent{}
+		result, err := Unmarshal(data, p)
+		if err != nil {
+			t.Errorf("unexpected error %v", err)
+		}
+		_, ok := result["nested"].(handleJSONDataChild)
+		if !ok {
+			t.Error("invalid map value")
+		}
+		if p.Nested.Data == nil {
+			t.Error("HandleJSONData not called")
+		}
+		if len(p.Nested.Data) != 2 || p.Nested.Data["known"] != "goo" || p.Nested.Data["unknown"] != "doo" {
+			t.Error("invalid JSON data")
+		}
+	})
+}
+
 var extraData = map[string]interface{}{
 	"extra1": "foo",
 	"extra2": float64(12),