From 6f514ac1b00afdcd08f6d2339a4d951be75f2cff Mon Sep 17 00:00:00 2001 From: Inteon <42113979+inteon@users.noreply.github.com> Date: Thu, 21 Oct 2021 15:07:45 +0200 Subject: [PATCH] use changes in sigs.k8s.io/json & match API-style Signed-off-by: Inteon <42113979+inteon@users.noreply.github.com> --- go.mod | 11 +- go.sum | 6 +- marshal.go | 43 +++++++ marshal_test.go | 35 ++++++ yaml.go => unmarshal.go | 179 +++++++++++++++++++----------- yaml_test.go => unmarshal_test.go | 97 ++++++---------- yaml_go110.go | 14 --- yaml_go110_test.go | 46 -------- 8 files changed, 238 insertions(+), 193 deletions(-) create mode 100644 marshal.go create mode 100644 marshal_test.go rename yaml.go => unmarshal.go (68%) rename yaml_test.go => unmarshal_test.go (82%) delete mode 100644 yaml_go110.go delete mode 100644 yaml_go110_test.go diff --git a/go.mod b/go.mod index 5fbe508..71fcd35 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,10 @@ -module github.com/amurant/yaml +module sigs.k8s.io/yaml -go 1.16 +go 1.17 -require gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b +require ( + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 +) + +replace gopkg.in/yaml.v3 => github.com/amurant/go-yaml v0.0.0-20211021125301-a69ce44590ee diff --git a/go.sum b/go.sum index e387ff0..72d32af 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ +github.com/amurant/go-yaml v0.0.0-20211021125301-a69ce44590ee h1:hh4GlzhK0sFfqNroFo5dgDa4NylwldjzlweQzm85sdI= +github.com/amurant/go-yaml v0.0.0-20211021125301-a69ce44590ee/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/marshal.go b/marshal.go new file mode 100644 index 0000000..08f0131 --- /dev/null +++ b/marshal.go @@ -0,0 +1,43 @@ +package yaml + +import ( + "fmt" + + "encoding/json" + + "gopkg.in/yaml.v3" +) + +// JSONToYAML Converts JSON to YAML. +func JSONToYAML(jsonBytes []byte) ([]byte, error) { + // Convert the JSON to an object. + var jsonObj interface{} + + // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the + // Go JSON library doesn't try to pick the right number type (int, float, + // etc.) when unmarshalling to interface{}, it just picks float64 + // universally. go-yaml does go through the effort of picking the right + // number type, so we can preserve number type throughout this process. + if err := yaml.Unmarshal(jsonBytes, &jsonObj); err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + // Marshal this object into YAML. + yamlBytes, err := yaml.Marshal(jsonObj) + if err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + return yamlBytes, nil +} + +// Marshal marshals the object into JSON then converts JSON to YAML and returns the +// YAML. +func Marshal(obj interface{}) ([]byte, error) { + jsonBytes, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("error marshaling into JSON: %v", err) + } + + return JSONToYAML(jsonBytes) +} diff --git a/marshal_test.go b/marshal_test.go new file mode 100644 index 0000000..a6027b3 --- /dev/null +++ b/marshal_test.go @@ -0,0 +1,35 @@ +package yaml + +import ( + "fmt" + "math" + "reflect" + "strconv" + "testing" +) + +func strPtr(str string) *string { + return &str +} + +type MarshalTest struct { + A string + B int64 + C float64 +} + +func TestMarshal(t *testing.T) { + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt64, math.MaxFloat64} + e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f64String)) + + y, err := Marshal(s) + if err != nil { + t.Errorf("error marshaling YAML: %v", err) + } + + if !reflect.DeepEqual(y, e) { + t.Errorf("marshal YAML was unsuccessful, expected: %#v, got: %#v", + string(e), string(y)) + } +} diff --git a/yaml.go b/unmarshal.go similarity index 68% rename from yaml.go rename to unmarshal.go index 90d049d..34da0aa 100644 --- a/yaml.go +++ b/unmarshal.go @@ -3,52 +3,63 @@ package yaml import ( "bytes" "encoding/json" + "errors" "fmt" "reflect" "strconv" + kubejson "sigs.k8s.io/json" + "gopkg.in/yaml.v3" ) -// JSONToYAML Converts JSON to YAML. -func JSONToYAML(jsonBytes []byte) ([]byte, error) { - // Convert the JSON to an object. - var jsonObj interface{} - - // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the - // Go JSON library doesn't try to pick the right number type (int, float, - // etc.) when unmarshalling to interface{}, it just picks float64 - // universally. go-yaml does go through the effort of picking the right - // number type, so we can preserve number type throughout this process. - if err := yaml.Unmarshal(jsonBytes, &jsonObj); err != nil { - return nil, fmt.Errorf("error converting JSON to YAML: %v", err) - } +type StrictOption = kubejson.StrictOption - // Marshal this object into YAML. - yamlBytes, err := yaml.Marshal(jsonObj) - if err != nil { - return nil, fmt.Errorf("error converting JSON to YAML: %v", err) - } +const ( + // DisallowDuplicateFields returns strict errors if data contains duplicate fields + DisallowDuplicateFields StrictOption = kubejson.DisallowDuplicateFields - return yamlBytes, nil -} + // DisallowUnknownFields returns strict errors if data contains unknown fields when decoding into typed structs + DisallowUnknownFields StrictOption = kubejson.DisallowUnknownFields +) -// Marshal marshals the object into JSON then converts JSON to YAML and returns the -// YAML. -func Marshal(obj interface{}) ([]byte, error) { - jsonBytes, err := json.Marshal(obj) - if err != nil { - return nil, fmt.Errorf("error marshaling into JSON: %v", err) - } +type yamlToTargetOption uint8 + +const ( + // no options enabled + none yamlToTargetOption = 0 - return JSONToYAML(jsonBytes) + // disallowDuplicateFields returns strict errors if data contains duplicate fields + disallowDuplicateFields yamlToTargetOption = 1 << iota + + // disallowUnknownFields returns strict errors if data contains unknown fields when decoding into typed structs + disallowUnknownFields + + // preserveNumberTyping preserves yaml types when jsonTarget is an interface type + preserveNumberTyping +) + +func yamlUnmarshal(yamlBytes []byte, v interface{}, options yamlToTargetOption) (strictErrors []error, err error) { + dec := yaml.NewDecoder(bytes.NewReader(yamlBytes)) + dec.KnownFields(options&disallowUnknownFields != 0) + dec.DisableUniqueKeys(options&disallowDuplicateFields != 0) + err = dec.Decode(v) + if e, ok := err.(*yaml.TypeError); ok { + errs := make([]error, 0, len(e.StrictErrors)) + for _, err := range e.StrictErrors { + errs = append(errs, errors.New(err)) + } + return errs, &yaml.TypeError{e.Errors, nil} + } + return nil, err } -func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, numberTyping bool) ([]byte, error) { +func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, options yamlToTargetOption) (jsonBytes []byte, strictErrors []error, err error) { // Decode yamlBytes into a yamlObj. var yamlObj interface{} - if err := yaml.Unmarshal(yamlBytes, &yamlObj); err != nil { - return nil, fmt.Errorf("error converting YAML to JSON: %v", err) + strictErrors, err = yamlUnmarshal(yamlBytes, &yamlObj, options) + if err != nil { + return nil, strictErrors, fmt.Errorf("error converting YAML to JSON: %v", err) } // Convert yamlObj to jsonObj. @@ -56,17 +67,17 @@ func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, numberTyping // can have non-string keys in YAML). So, convert the YAML-compatible object // to a JSON-compatible object, failing with an error if irrecoverable // incompatibilities happen along the way. - jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget, numberTyping) + jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget, options&preserveNumberTyping != 0) if err != nil { - return nil, fmt.Errorf("error converting YAML to JSON: %v", err) + return nil, strictErrors, fmt.Errorf("error converting YAML to JSON: %v", err) } // Encode a jsonObj into jsonBytes. - jsonBytes, err := json.Marshal(jsonObj) + jsonBytes, err = json.Marshal(jsonObj) if err != nil { - return nil, fmt.Errorf("error converting YAML to JSON: %v", err) + return nil, strictErrors, fmt.Errorf("error converting YAML to JSON: %v", err) } - return jsonBytes, nil + return jsonBytes, strictErrors, nil } // YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, @@ -82,56 +93,96 @@ func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, numberTyping // // For strict decoding of YAML, use YAMLToJSONStrict. func YAMLToJSON(yamlBytes []byte) ([]byte, error) { - return yamlToJSONTarget(yamlBytes, nil, false) + json, _, err := yamlToJSONTarget(yamlBytes, nil, none) + return json, err } // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, // returning an error on any duplicate field names. // Deprecated: identical to YAMLToJSON -func YAMLToJSONStrict(yamlBytes []byte) ([]byte, error) { - return YAMLToJSON(yamlBytes) +func YAMLToJSONStrict(yamlBytes []byte, strictOptions ...StrictOption) (json []byte, strictErrors []error, err error) { + options := none + if len(strictOptions) > 0 { + for _, strictOpt := range strictOptions { + switch strictOpt { + case DisallowDuplicateFields: + options = options | disallowDuplicateFields + case DisallowUnknownFields: + options = options | disallowUnknownFields + default: + return nil, nil, fmt.Errorf("unknown strict option %d", strictOpt) + } + } + } else { + options = none | disallowDuplicateFields | disallowUnknownFields + } + return yamlToJSONTarget(yamlBytes, nil, options) } -// JSONOpt is a decoding option for decoding from JSON format. -type JSONOpt func(*json.Decoder) *json.Decoder - -// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, -// optionally configuring the behavior of the JSON unmarshal. -func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { +func unmarshal(yamlBytes []byte, obj interface{}, options yamlToTargetOption) (strictErrors []error, err error) { jsonTarget := reflect.ValueOf(obj) if jsonTarget.Kind() != reflect.Ptr || jsonTarget.IsNil() { - return fmt.Errorf("provided object is not a valid pointer") + return nil, fmt.Errorf("provided object is not a valid pointer") } - jsonBytes, err := yamlToJSONTarget(yamlBytes, &jsonTarget, true) + jsonBytes, strictErrors, err := yamlToJSONTarget(yamlBytes, &jsonTarget, options) if err != nil { - return err + return strictErrors, err } // Decode jsonBytes into obj. - jsonDecoder := json.NewDecoder(bytes.NewReader(jsonBytes)) - for _, opt := range opts { - jsonDecoder = opt(jsonDecoder) + strictOptions := []StrictOption{} + if options&disallowDuplicateFields != 0 { + strictOptions = append(strictOptions, DisallowDuplicateFields) } - if err := jsonDecoder.Decode(&obj); err != nil { - return fmt.Errorf("error unmarshaling JSON: while decoding JSON: %v", err) + if options&disallowUnknownFields != 0 { + strictOptions = append(strictOptions, DisallowUnknownFields) } - - pv := jsonTarget.Elem() - val, err := convertNestedJSONNumbers(pv.Interface()) + strictErrors2, err := kubejson.UnmarshalStrict(jsonBytes, &obj, strictOptions...) + strictErrors = append(strictErrors, strictErrors2...) if err != nil { - return err + return strictErrors, fmt.Errorf("error unmarshaling JSON: while decoding JSON: %v", err) + } + + if options&preserveNumberTyping != 0 { + pv := jsonTarget.Elem() + val, err := convertNestedJSONNumbers(pv.Interface()) + if err != nil { + return strictErrors, err + } + pv.Set(reflect.ValueOf(val)) } - pv.Set(reflect.ValueOf(val)) - return nil + return strictErrors, nil +} + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(yamlBytes []byte, obj interface{}) error { + _, err := unmarshal(yamlBytes, obj, preserveNumberTyping) + return err } // UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal // into an object, optionally configuring the behavior of the JSON unmarshal. // Deprecated: identical to Unmarshal -func UnmarshalStrict(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return Unmarshal(yamlBytes, obj, opts...) +func UnmarshalStrict(yamlBytes []byte, obj interface{}, strictOptions ...StrictOption) (strictErrors []error, err error) { + options := preserveNumberTyping + if len(strictOptions) > 0 { + for _, strictOpt := range strictOptions { + switch strictOpt { + case DisallowDuplicateFields: + options = options | disallowDuplicateFields + case DisallowUnknownFields: + options = options | disallowUnknownFields + default: + return nil, fmt.Errorf("unknown strict option %d", strictOpt) + } + } + } else { + options = options | disallowDuplicateFields | disallowUnknownFields + } + return unmarshal(yamlBytes, obj, options) } func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value, numberTyping bool) (interface{}, error) { @@ -174,6 +225,7 @@ cast_type: // keys can only have the types string, int, int64, float64, binary // (unsupported), or null (unsupported). strMap := make(map[string]interface{}) + insensitive := false for k, v := range typedYAMLObj { // Resolve the key to a string first. var keyString string @@ -201,12 +253,14 @@ cast_type: s = ".nan" } keyString = s + insensitive = true case bool: if typedKey { keyString = "true" } else { keyString = "false" } + insensitive = true default: return nil, fmt.Errorf("unsupported map key of type: %s, key: %+#v, value: %+#v", reflect.TypeOf(k), k, v) @@ -231,7 +285,8 @@ cast_type: break } // Do case-insensitive comparison. - if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { + if f == nil && insensitive && ff.equalFold(ff.nameBytes, keyBytes) { + keyString = string(ff.nameBytes) f = ff } } diff --git a/yaml_test.go b/unmarshal_test.go similarity index 82% rename from yaml_test.go rename to unmarshal_test.go index 76b7007..f848fa1 100644 --- a/yaml_test.go +++ b/unmarshal_test.go @@ -1,39 +1,10 @@ package yaml import ( - "fmt" - "math" "reflect" - "strconv" "testing" ) -func strPtr(str string) *string { - return &str -} - -type MarshalTest struct { - A string - B int64 - C float64 -} - -func TestMarshal(t *testing.T) { - f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) - s := MarshalTest{"a", math.MaxInt64, math.MaxFloat64} - e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f64String)) - - y, err := Marshal(s) - if err != nil { - t.Errorf("error marshaling YAML: %v", err) - } - - if !reflect.DeepEqual(y, e) { - t.Errorf("marshal YAML was unsuccessful, expected: %#v, got: %#v", - string(e), string(y)) - } -} - type NestedString struct { A string } @@ -88,6 +59,10 @@ type WithPtrEmbed struct { B string `json:"b"` } +type WithTaggedField struct { + Field string `json:"field"` +} + type unmarshalTestCase struct { encoded []byte decodeInto interface{} @@ -97,12 +72,12 @@ type unmarshalTestCase struct { func TestUnmarshal(t *testing.T) { tests := []unmarshalTestCase{ { - encoded: []byte("a: 1"), + encoded: []byte("A: 1"), decodeInto: new(UnmarshalString), decoded: UnmarshalString{A: "1"}, }, { - encoded: []byte("a: true"), + encoded: []byte("A: true"), decodeInto: new(UnmarshalString), decoded: UnmarshalString{A: "true"}, }, @@ -112,20 +87,25 @@ func TestUnmarshal(t *testing.T) { decoded: UnmarshalString{True: "1"}, }, { - encoded: []byte("a:\n a: 1"), - decodeInto: new(UnmarshalNestedString), - decoded: UnmarshalNestedString{A: NestedString{A: "1"}}, + encoded: []byte("True: 1"), + decodeInto: new(UnmarshalString), + decoded: UnmarshalString{True: "1"}, }, { - encoded: []byte("a:\n - b: abc\n c: def\n - b: 123\n c: 456\n"), - decodeInto: new(UnmarshalSlice), - decoded: UnmarshalSlice{A: []NestedSlice{{B: "abc", C: strPtr("def")}, {B: "123", C: strPtr("456")}}}, + encoded: []byte("A:\n A: 1"), + decodeInto: new(UnmarshalNestedString), + decoded: UnmarshalNestedString{NestedString{"1"}}, }, { - encoded: []byte("a:\n b: 1"), + encoded: []byte("A:\n b: 1"), decodeInto: new(UnmarshalStringMap), decoded: UnmarshalStringMap{map[string]string{"b": "1"}}, }, + { + encoded: []byte("A:\n - B: abc\n C: def\n - B: 123\n C: 456\n"), + decodeInto: new(UnmarshalSlice), + decoded: UnmarshalSlice{[]NestedSlice{{"abc", strPtr("def")}, {"123", strPtr("456")}}}, + }, { encoded: []byte("a:\n name: TestA\nb:\n name: TestB"), decodeInto: new(map[string]*NamedThing), @@ -134,31 +114,6 @@ func TestUnmarshal(t *testing.T) { "b": {Name: "TestB"}, }, }, - { - encoded: []byte("a: true"), - decodeInto: new(UnmarshalString), - decoded: UnmarshalString{A: "true"}, - }, - { - encoded: []byte("true: 1"), - decodeInto: new(UnmarshalString), - decoded: UnmarshalString{True: "1"}, - }, - { - encoded: []byte("a:\n a: 1"), - decodeInto: new(UnmarshalNestedString), - decoded: UnmarshalNestedString{NestedString{"1"}}, - }, - { - encoded: []byte("a:\n - b: abc\n c: def\n - b: 123\n c: 456\n"), - decodeInto: new(UnmarshalSlice), - decoded: UnmarshalSlice{[]NestedSlice{{"abc", strPtr("def")}, {"123", strPtr("456")}}}, - }, - { - encoded: []byte("a:\n b: 1"), - decodeInto: new(UnmarshalStringMap), - decoded: UnmarshalStringMap{map[string]string{"b": "1"}}, - }, { encoded: []byte("a:\n name: TestA\nb:\n name: TestB"), decodeInto: new(map[string]*NamedThing2), @@ -250,6 +205,16 @@ func TestUnmarshal(t *testing.T) { "b": nil, }, }, + { + encoded: []byte(`field: "hello"`), + decodeInto: new(WithTaggedField), + decoded: WithTaggedField{Field: "hello"}, + }, + { + encoded: []byte(`unknown: "hello"`), + decodeInto: new(WithTaggedField), + decoded: nil, + }, } for _, test := range tests { @@ -272,12 +237,12 @@ func TestUnmarshal(t *testing.T) { continue } - err := Unmarshal(test.encoded, value.Interface()) - if err != nil { + strictErrors, err := UnmarshalStrict(test.encoded, value.Interface()) + if err != nil || len(strictErrors) > 0 { if test.decoded == nil { continue } - t.Errorf("error unmarshaling YAML: %v", err) + t.Errorf("error unmarshaling YAML: %v %v", err, strictErrors) } if !reflect.DeepEqual(value.Elem().Interface(), test.decoded) { diff --git a/yaml_go110.go b/yaml_go110.go deleted file mode 100644 index ab3e06a..0000000 --- a/yaml_go110.go +++ /dev/null @@ -1,14 +0,0 @@ -// This file contains changes that are only compatible with go 1.10 and onwards. - -// +build go1.10 - -package yaml - -import "encoding/json" - -// DisallowUnknownFields configures the JSON decoder to error out if unknown -// fields come along, instead of dropping them by default. -func DisallowUnknownFields(d *json.Decoder) *json.Decoder { - d.DisallowUnknownFields() - return d -} diff --git a/yaml_go110_test.go b/yaml_go110_test.go deleted file mode 100644 index e4707e8..0000000 --- a/yaml_go110_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// +build go1.10 - -package yaml - -import ( - "fmt" - "testing" -) - -func TestUnmarshalWithTags(t *testing.T) { - type WithTaggedField struct { - Field string `json:"field"` - } - - t.Run("Known tagged field", func(t *testing.T) { - y := []byte(`field: "hello"`) - v := WithTaggedField{} - if err := Unmarshal(y, &v, DisallowUnknownFields); err != nil { - t.Errorf("unexpected error: %v", err) - } - if v.Field != "hello" { - t.Errorf("v.Field=%v, want 'hello'", v.Field) - } - - }) - t.Run("With unknown tagged field", func(t *testing.T) { - y := []byte(`unknown: "hello"`) - v := WithTaggedField{} - err := Unmarshal(y, &v, DisallowUnknownFields) - if err == nil { - t.Errorf("want error because of unknown field, got : v=%#v", v) - } - }) - -} - -func exampleUnknown() { - type WithTaggedField struct { - Field string `json:"field"` - } - y := []byte(`unknown: "hello"`) - v := WithTaggedField{} - fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields)) - // Ouptut: - // unmarshaling JSON: while decoding JSON: json: unknown field "unknown" -}