From efe7d3f45ea2d6a7e78d8b7bdd09739814816afa Mon Sep 17 00:00:00 2001 From: Inteon <42113979+inteon@users.noreply.github.com> Date: Fri, 5 Nov 2021 18:30:55 +0100 Subject: [PATCH] switch to 'sigs.k8s.io/json' and 'gopkg.in/yaml.v3' & cleanup/ update fields.go & update interface --- .github/workflows/go.yml | 2 +- fields.go | 285 +++++---------------- go.mod | 7 +- go.sum | 14 +- yaml.go | 227 +++++++---------- yaml_go110.go | 31 --- yaml_go110_test.go | 63 ----- yaml_test.go | 527 ++++++++++++++++----------------------- 8 files changed, 393 insertions(+), 763 deletions(-) delete mode 100644 yaml_go110.go delete mode 100644 yaml_go110_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 07fc106..cb50479 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -9,7 +9,7 @@ jobs: build: strategy: matrix: - go-versions: [1.13.x, 1.14.x, 1.15.x, 1.16.x, 1.17.x] + go-versions: [1.17.x] runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 diff --git a/fields.go b/fields.go index f3c9079..14d0759 100644 --- a/fields.go +++ b/fields.go @@ -5,7 +5,6 @@ package yaml import ( - "bytes" "encoding" "encoding/json" "reflect" @@ -13,26 +12,25 @@ import ( "strings" "sync" "unicode" - "unicode/utf8" ) // indirect walks down 'value' allocating pointers as needed, // until it gets to a non-pointer. -// if it encounters an Unmarshaler, indirect stops and returns that. -// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. -func indirect(value reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { +// If it encounters an Unmarshaler, indirect stops and returns nil. +func indirect(value reflect.Value) *reflect.Value { // If 'value' is a named type and is addressable, // start with its address, so that if the type has pointer methods, // we find them. if value.Kind() != reflect.Ptr && value.Type().Name() != "" && value.CanAddr() { value = value.Addr() } + for { // Load value from interface, but only if the result will be // usefully addressable. if value.Kind() == reflect.Interface && !value.IsNil() { element := value.Elem() - if element.Kind() == reflect.Ptr && !element.IsNil() && (!decodingNull || element.Elem().Kind() == reflect.Ptr) { + if element.Kind() == reflect.Ptr && !element.IsNil() { value = element continue } @@ -42,34 +40,38 @@ func indirect(value reflect.Value, decodingNull bool) (json.Unmarshaler, encodin break } - if value.Elem().Kind() != reflect.Ptr && decodingNull && value.CanSet() { + // Prevent infinite loop if v is an interface pointing to its own address: + // var v interface{} + // v = &v + if value.Elem().Kind() == reflect.Interface && value.Elem().Elem() == value { + value = value.Elem() break } + if value.IsNil() { - if value.CanSet() { - value.Set(reflect.New(value.Type().Elem())) - } else { - value = reflect.New(value.Type().Elem()) - } + value = reflect.New(value.Type().Elem()) } - if value.Type().NumMethod() > 0 { - if u, ok := value.Interface().(json.Unmarshaler); ok { - return u, nil, reflect.Value{} + + // We have a JSON or Text Umarshaler at this level, so we can't be trying + // to decode into a string. + if value.Type().NumMethod() > 0 && value.CanInterface() { + if _, ok := value.Interface().(json.Unmarshaler); ok { + return nil } - if u, ok := value.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, reflect.Value{} + if _, ok := value.Interface().(encoding.TextUnmarshaler); ok { + return nil } } + value = value.Elem() } - return nil, nil, value + + return &value } // A field represents a single field found in a struct. type field struct { - name string - nameBytes []byte // []byte(name) - equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + name string tag bool index []int @@ -78,12 +80,6 @@ type field struct { quoted bool } -func fillField(f field) field { - f.nameBytes = []byte(f.name) - f.equalFold = foldFunc(f.nameBytes) - return f -} - // byName sorts field by name, breaking ties with depth, // then breaking ties with "name came from json tag", then // breaking ties with index sequence. @@ -134,8 +130,7 @@ func typeFields(t reflect.Type) []field { next := []field{{typ: t}} // Count of queued names for current level and the next. - var count map[reflect.Type]int - var nextCount map[reflect.Type]int + var count, nextCount map[reflect.Type]int // Types already visited at an earlier level. visited := map[reflect.Type]bool{} @@ -156,7 +151,19 @@ func typeFields(t reflect.Type) []field { // Scan f.typ for fields to include. for i := 0; i < f.typ.NumField(); i++ { sf := f.typ.Field(i) - if sf.PkgPath != "" { // unexported + if sf.Anonymous { + t := sf.Type + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if !sf.IsExported() && t.Kind() != reflect.Struct { + // Ignore embedded fields of unexported non-struct types. + continue + } + // Do not ignore embedded fields of unexported struct types + // since they may have exported fields. + } else if !sf.IsExported() { + // Ignore unexported non-embedded fields. continue } tag := sf.Tag.Get("json") @@ -177,20 +184,34 @@ func typeFields(t reflect.Type) []field { ft = ft.Elem() } + // Only strings, floats, integers, and booleans can be quoted. + quoted := false + if opts.Contains("string") { + switch ft.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, + reflect.String: + quoted = true + } + } + // Record found field and index sequence. if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { tagged := name != "" if name == "" { name = sf.Name } - fields = append(fields, fillField(field{ + fields = append(fields, field{ name: name, tag: tagged, index: index, typ: ft, omitEmpty: opts.Contains("omitempty"), - quoted: opts.Contains("string"), - })) + quoted: quoted, + }) + if count[f.typ] > 1 { // If there were multiple instances, add a second, // so that the annihilation code will see a duplicate. @@ -204,7 +225,7 @@ func typeFields(t reflect.Type) []field { // Record new anonymous struct to explore in next round. nextCount[ft]++ if nextCount[ft] == 1 { - next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + next = append(next, field{name: ft.Name(), index: index, typ: ft}) } } } @@ -253,65 +274,24 @@ func typeFields(t reflect.Type) []field { // will be false: This condition is an error in Go and we skip all // the fields. func dominantField(fields []field) (field, bool) { - // The fields are sorted in increasing index-length order. The winner - // must therefore be one with the shortest index length. Drop all - // longer entries, which is easy: just truncate the slice. - length := len(fields[0].index) - tagged := -1 // Index of first tagged field. - for i, f := range fields { - if len(f.index) > length { - fields = fields[:i] - break - } - if f.tag { - if tagged >= 0 { - // Multiple tagged fields at the same level: conflict. - // Return no field. - return field{}, false - } - tagged = i - } - } - if tagged >= 0 { - return fields[tagged], true - } - // All remaining fields have the same length. If there's more than one, - // we have a conflict (two fields named "X" at the same level) and we - // return no field. - if len(fields) > 1 { + // The fields are sorted in increasing index-length order, then by presence of tag. + // That means that the first field is the dominant one. We need only check + // for error cases: two fields at top level, either both tagged or neither tagged. + if len(fields) > 1 && len(fields[0].index) == len(fields[1].index) && fields[0].tag == fields[1].tag { return field{}, false } return fields[0], true } -var fieldCache struct { - sync.RWMutex - m map[reflect.Type][]field -} +var fieldCache sync.Map // map[reflect.Type]structFields // cachedTypeFields is like typeFields but uses a cache to avoid repeated work. func cachedTypeFields(t reflect.Type) []field { - fieldCache.RLock() - f := fieldCache.m[t] - fieldCache.RUnlock() - if f != nil { - return f - } - - // Compute fields without lock. - // Might duplicate effort but won't hold other computations back. - f = typeFields(t) - if f == nil { - f = []field{} + if f, ok := fieldCache.Load(t); ok { + return f.([]field) } - - fieldCache.Lock() - if fieldCache.m == nil { - fieldCache.m = map[reflect.Type][]field{} - } - fieldCache.m[t] = f - fieldCache.Unlock() - return f + f, _ := fieldCache.LoadOrStore(t, typeFields(t)) + return f.([]field) } func isValidTag(s string) bool { @@ -320,144 +300,11 @@ func isValidTag(s string) bool { } for _, c := range s { switch { - case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + case strings.ContainsRune("!#$%&()*+-./:;<=>?@[]^_{|}~ ", c): // Backslash and quote chars are reserved, but // otherwise any punctuation chars are allowed // in a tag name. - default: - if !unicode.IsLetter(c) && !unicode.IsDigit(c) { - return false - } - } - } - return true -} - -const ( - caseMask = ^byte(0x20) // Mask to ignore case in ASCII. - kelvin = '\u212a' - smallLongEss = '\u017f' -) - -// foldFunc returns one of four different case folding equivalence -// functions, from most general (and slow) to fastest: -// -// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 -// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') -// 3) asciiEqualFold, no special, but includes non-letters (including _) -// 4) simpleLetterEqualFold, no specials, no non-letters. -// -// The letters S and K are special because they map to 3 runes, not just 2: -// * S maps to s and to U+017F 'ſ' Latin small letter long s -// * k maps to K and to U+212A 'K' Kelvin sign -// See http://play.golang.org/p/tTxjOc0OGo -// -// The returned function is specialized for matching against s and -// should only be given s. It's not curried for performance reasons. -func foldFunc(s []byte) func(s, t []byte) bool { - nonLetter := false - special := false // special letter - for _, b := range s { - if b >= utf8.RuneSelf { - return bytes.EqualFold - } - upper := b & caseMask - if upper < 'A' || upper > 'Z' { - nonLetter = true - } else if upper == 'K' || upper == 'S' { - // See above for why these letters are special. - special = true - } - } - if special { - return equalFoldRight - } - if nonLetter { - return asciiEqualFold - } - return simpleLetterEqualFold -} - -// equalFoldRight is a specialization of bytes.EqualFold when s is -// known to be all ASCII (including punctuation), but contains an 's', -// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. -// See comments on foldFunc. -func equalFoldRight(s, t []byte) bool { - for _, sb := range s { - if len(t) == 0 { - return false - } - tb := t[0] - if tb < utf8.RuneSelf { - if sb != tb { - sbUpper := sb & caseMask - if 'A' <= sbUpper && sbUpper <= 'Z' { - if sbUpper != tb&caseMask { - return false - } - } else { - return false - } - } - t = t[1:] - continue - } - // sb is ASCII and t is not. t must be either kelvin - // sign or long s; sb must be s, S, k, or K. - tr, size := utf8.DecodeRune(t) - switch sb { - case 's', 'S': - if tr != smallLongEss { - return false - } - case 'k', 'K': - if tr != kelvin { - return false - } - default: - return false - } - t = t[size:] - - } - - return len(t) <= 0 -} - -// asciiEqualFold is a specialization of bytes.EqualFold for use when -// s is all ASCII (but may contain non-letters) and contains no -// special-folding letters. -// See comments on foldFunc. -func asciiEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, sb := range s { - tb := t[i] - if sb == tb { - continue - } - if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { - if sb&caseMask != tb&caseMask { - return false - } - } else { - return false - } - } - return true -} - -// simpleLetterEqualFold is a specialization of bytes.EqualFold for -// use when s is all ASCII letters (no underscores, etc) and also -// doesn't contain 'k', 'K', 's', or 'S'. -// See comments on foldFunc. -func simpleLetterEqualFold(s, t []byte) bool { - if len(s) != len(t) { - return false - } - for i, b := range s { - if b&caseMask != t[i]&caseMask { + case !unicode.IsLetter(c) && !unicode.IsDigit(c): return false } } diff --git a/go.mod b/go.mod index 818bbb5..b193b7f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,11 @@ module sigs.k8s.io/yaml -go 1.12 +go 1.17 require ( - github.com/davecgh/go-spew v1.1.1 gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 ) + +require gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index b7b8cbb..4e823f4 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,14 @@ -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/yaml.go b/yaml.go index 04ed20f..21ec7ce 100644 --- a/yaml.go +++ b/yaml.go @@ -20,15 +20,35 @@ import ( "bytes" "encoding/json" "fmt" - "io" "reflect" "strconv" - "gopkg.in/yaml.v2" + kubejson "sigs.k8s.io/json" + + yamlv2 "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" +) + +type disallowUnknownFieldsOption bool + +const ( + // disallowUnknownFields returns strict errors if data contains unknown fields when decoding into typed structs + disallowUnknownFields disallowUnknownFieldsOption = true + + // allowUnknownFields returns no errors if data contains unknown fields when decoding into typed structs + allowUnknownFields disallowUnknownFieldsOption = false ) // Marshal marshals obj into JSON using stdlib json.Marshal, and then converts JSON to YAML using JSONToYAML (see that method for more reference) func Marshal(obj interface{}) ([]byte, error) { + if yamlNode, ok := obj.(*yaml.Node); ok { + var buf bytes.Buffer + if err := yamlNode.Decode(&buf); err != nil { + return nil, err + } + return buf.Bytes(), nil + } + jsonBytes, err := json.Marshal(obj) if err != nil { return nil, fmt.Errorf("error marshaling into JSON: %w", err) @@ -37,84 +57,76 @@ func Marshal(obj interface{}) ([]byte, error) { return JSONToYAML(jsonBytes) } -// JSONOpt is a decoding option for decoding from JSON format. -type JSONOpt func(*json.Decoder) *json.Decoder - // Unmarshal first converts the given YAML to JSON, and then unmarshals the JSON into obj. Options for the // standard library json.Decoder can be optionally specified, e.g. to decode untyped numbers into json.Number instead of float64, or to disallow unknown fields (but for that purpose, see also UnmarshalStrict). obj must be a non-nil pointer. // // Important notes about the Unmarshal logic: // -// - Decoding is case-insensitive, unlike the rest of Kubernetes API machinery, as this is using the stdlib json library. This might be confusing to users. -// - This decodes any number (although it is an integer) into a float64 if the type of obj is unknown, e.g. *map[string]interface{}, *interface{}, or *[]interface{}. This means integers above +/- 2^53 will lose precision when round-tripping. Make a JSONOpt that calls d.UseNumber() to avoid this. -// - Duplicate fields, including in-case-sensitive matches, are ignored in an undefined order. Note that the YAML specification forbids duplicate fields, so this logic is more permissive than it needs to. See UnmarshalStrict for an alternative. -// - Unknown fields, i.e. serialized data that do not map to a field in obj, are ignored. Use d.DisallowUnknownFields() or UnmarshalStrict to override. -// - As per the YAML 1.1 specification, which yaml.v2 used underneath implements, literal 'yes' and 'no' strings without quotation marks will be converted to true/false implicitly. +// - Decoding is case-sensitive, just like the rest of the Kubernetes API machinery decoder logic, but unlike the standard library JSON encoder. +// - Duplicate fields (only case-sensitive matches) in objects result in a fatal error, as defined in the YAML spec. +// - The sequence indentation style is wide, which means that the "- " marker for a YAML sequence will NOT be on the same indentation level as the sequence field name, but two spaces more indented. +// - Unknown fields, i.e. serialized data that do not map to a field in obj, are ignored. Use UnmarshalStrict to override. // - YAML non-string keys, e.g. ints, bools and floats, are converted to strings implicitly during the YAML to JSON conversion process. // - There are no compatibility guarantees for returned error values. -func Unmarshal(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.Unmarshal, opts...) +func Unmarshal(yamlBytes []byte, obj interface{}) error { + _, err := unmarshal(yamlBytes, obj, allowUnknownFields) + return err } // UnmarshalStrict is similar to Unmarshal (please read its documentation for reference), with the following exceptions: // -// - Duplicate fields in an object yield an error. This is according to the YAML specification. -// - If obj, or any of its recursive children, is a struct, presence of fields in the serialized data unknown to the struct will yield an error. -func UnmarshalStrict(yamlBytes []byte, obj interface{}, opts ...JSONOpt) error { - return unmarshal(yamlBytes, obj, yaml.UnmarshalStrict, append(opts, DisallowUnknownFields)...) +// - If obj, or any of its recursive children, is a struct, presence of fields in the serialized data unknown to the struct will yield a strict error. +func UnmarshalStrict(yamlBytes []byte, obj interface{}) (strictErrors []error, err error) { + return unmarshal(yamlBytes, obj, disallowUnknownFields) } // unmarshal unmarshals the given YAML byte stream into the given interface, // optionally performing the unmarshalling strictly -func unmarshal(yamlBytes []byte, obj interface{}, unmarshalFn func([]byte, interface{}) error, opts ...JSONOpt) error { - jsonTarget := reflect.ValueOf(obj) +func unmarshal(yamlBytes []byte, obj interface{}, unknownFieldsOption disallowUnknownFieldsOption) (strictErrors []error, err error) { + if yamlNode, ok := obj.(*yaml.Node); ok { + if err := yamlNode.Encode(yamlBytes); err != nil { + return nil, err + } + return nil, nil + } - jsonBytes, err := yamlToJSONTarget(yamlBytes, &jsonTarget, unmarshalFn) - if err != nil { - return fmt.Errorf("error converting YAML to JSON: %w", err) + jsonTarget := reflect.ValueOf(obj) + if jsonTarget.Kind() != reflect.Ptr || jsonTarget.IsNil() { + return nil, fmt.Errorf("provided object is not a valid pointer") } - err = jsonUnmarshal(bytes.NewReader(jsonBytes), obj, opts...) + jsonBytes, err := yamlToJSONTarget(yamlBytes, &jsonTarget) if err != nil { - return fmt.Errorf("error unmarshaling JSON: %w", err) + return nil, err } - return nil -} + // Decode jsonBytes into obj. + if unknownFieldsOption == disallowUnknownFields { + strictErrors, err = kubejson.UnmarshalStrict(jsonBytes, &obj, kubejson.DisallowUnknownFields) + } else { + strictErrors, err = kubejson.UnmarshalStrict(jsonBytes, &obj) + } -// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the -// object, optionally applying decoder options prior to decoding. We are not -// using json.Unmarshal directly as we want the chance to pass in non-default -// options. -func jsonUnmarshal(reader io.Reader, obj interface{}, opts ...JSONOpt) error { - d := json.NewDecoder(reader) - for _, opt := range opts { - d = opt(d) + if err != nil { + return strictErrors, fmt.Errorf("error unmarshaling JSON: %w", err) } - return d.Decode(obj) + + return strictErrors, nil } // JSONToYAML converts JSON to YAML. Notable implementation details: // -// - Duplicate fields, are case-sensitively ignored in an undefined order. -// - The sequence indentation style is compact, which means that the "- " marker for a YAML sequence will be on the same indentation level as the sequence field name. -// - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. -func JSONToYAML(j []byte) ([]byte, error) { +// - Duplicate fields (only case-sensitive matches) in objects result in a fatal error, as defined in the YAML spec. +func JSONToYAML(jsonBytes []byte) ([]byte, error) { // Convert the JSON to an object. var jsonObj interface{} - // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the - // Go JSON library doesn't try to pick the right number type (int, float, - // etc.) when unmarshalling to interface{}, it just picks float64 - // universally. go-yaml does go through the effort of picking the right - // number type, so we can preserve number type throughout this process. - err := yaml.Unmarshal(j, &jsonObj) - if err != nil { + if err := kubejson.UnmarshalCaseSensitivePreserveInts(jsonBytes, &jsonObj); err != nil { return nil, fmt.Errorf("error converting JSON to YAML: %w", err) } // Marshal this object into YAML. - yamlBytes, err := yaml.Marshal(jsonObj) + yamlBytes, err := yamlv2.Marshal(jsonObj) if err != nil { return nil, fmt.Errorf("error converting JSON to YAML: %w", err) } @@ -136,24 +148,16 @@ func JSONToYAML(j []byte) ([]byte, error) { // // Notable about the implementation: // -// - Duplicate fields are case-sensitively ignored in an undefined order. Note that the YAML specification forbids duplicate fields, so this logic is more permissive than it needs to. See YAMLToJSONStrict for an alternative. -// - As per the YAML 1.1 specification, which yaml.v2 used underneath implements, literal 'yes' and 'no' strings without quotation marks will be converted to true/false implicitly. -// - Unlike Unmarshal, all integers, up to 64 bits, are preserved during this round-trip. +// - Duplicate fields (only case-sensitive matches) in objects result in a fatal error, as defined in the YAML spec. // - There are no compatibility guarantees for returned error values. -func YAMLToJSON(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.Unmarshal) -} - -// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, -// returning an error on any duplicate field names. -func YAMLToJSONStrict(y []byte) ([]byte, error) { - return yamlToJSONTarget(y, nil, yaml.UnmarshalStrict) +func YAMLToJSON(yamlBytes []byte) ([]byte, error) { + return yamlToJSONTarget(yamlBytes, nil) } -func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, unmarshalFn func([]byte, interface{}) error) ([]byte, error) { +func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value) (jsonBytes []byte, err error) { // Convert the YAML to an object. var yamlObj interface{} - err := unmarshalFn(yamlBytes, &yamlObj) + err = yaml.Unmarshal(yamlBytes, &yamlObj) if err != nil { return nil, fmt.Errorf("error converting YAML to JSON: %w", err) } @@ -168,7 +172,7 @@ func yamlToJSONTarget(yamlBytes []byte, jsonTarget *reflect.Value, unmarshalFn f } // Convert this object to JSON and return the data. - jsonBytes, err := json.Marshal(jsonObj) + jsonBytes, err = json.Marshal(jsonObj) if err != nil { return nil, fmt.Errorf("error converting YAML to JSON: %w", err) } @@ -183,14 +187,16 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in // decoding into the value, we're just checking if the ultimate target is a // string. if jsonTarget != nil { - jsonUnmarshaler, textUnmarshaler, pointerValue := indirect(*jsonTarget, false) - // We have a JSON or Text Umarshaler at this level, so we can't be trying - // to decode into a string. - if jsonUnmarshaler != nil || textUnmarshaler != nil { - jsonTarget = nil - } else { - jsonTarget = &pointerValue + jsonTarget = indirect(*jsonTarget) + } + + // Transform map[string]interface{} into map[interface{}]interface{} + if stringMap, ok := yamlObj.(map[string]interface{}); ok { + interfaceMap := make(map[interface{}]interface{}) + for k, v := range stringMap { + interfaceMap[k] = v } + yamlObj = interfaceMap } // If yamlObj is a number or a boolean, check if jsonTarget is a string - @@ -253,25 +259,30 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in if jsonTarget != nil { t := *jsonTarget if t.Kind() == reflect.Struct { - keyBytes := []byte(keyString) // Find the field that the JSON library would use. var f *field fields := cachedTypeFields(t.Type()) for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, keyBytes) { - f = ff + f = &fields[i] + if f.name == keyString { break } - // Do case-insensitive comparison. - if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { - f = ff - } } + if f != nil { // Find the reflect.Value of the most preferential // struct field. - jtf := t.Field(f.index[0]) + jtf := t + for _, i := range f.index { + if jtf.Kind() == reflect.Ptr { + if jtf.IsNil() { + jtf = reflect.New(jtf.Type().Elem()) + } + jtf = jtf.Elem() + } + jtf = jtf.Field(i) + } + strMap[keyString], err = convertToJSONableObject(v, &jtf) if err != nil { return nil, err @@ -336,7 +347,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in case int64: s = strconv.FormatInt(typedVal, 10) case float64: - s = strconv.FormatFloat(typedVal, 'g', -1, 32) + s = strconv.FormatFloat(typedVal, 'g', -1, 64) case uint64: s = strconv.FormatUint(typedVal, 10) case bool: @@ -350,67 +361,7 @@ func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (in yamlObj = interface{}(s) } } - return yamlObj, nil - } -} -// JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice, -// without going through a byte representation. A nil or empty map[string]interface{} input is -// converted to an empty map, i.e. yaml.MapSlice(nil). -// -// interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice. -// -// int64 and float64 are down casted following the logic of github.com/go-yaml/yaml: -// - float64s are down-casted as far as possible without data-loss to int, int64, uint64. -// - int64s are down-casted to int if possible without data-loss. -// -// Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. -// -// string, bool and any other types are unchanged. -func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { - if len(j) == 0 { - return nil - } - ret := make(yaml.MapSlice, 0, len(j)) - for k, v := range j { - ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) - } - return ret -} - -func jsonToYAMLValue(j interface{}) interface{} { - switch j := j.(type) { - case map[string]interface{}: - if j == nil { - return interface{}(nil) - } - return JSONObjectToYAMLObject(j) - case []interface{}: - if j == nil { - return interface{}(nil) - } - ret := make([]interface{}, len(j)) - for i := range j { - ret[i] = jsonToYAMLValue(j[i]) - } - return ret - case float64: - // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151 - if i64 := int64(j); j == float64(i64) { - if i := int(i64); i64 == int64(i) { - return i - } - return i64 - } - if ui64 := uint64(j); j == float64(ui64) { - return ui64 - } - return j - case int64: - if i := int(j); j == int64(i) { - return i - } - return j + return yamlObj, nil } - return j } diff --git a/yaml_go110.go b/yaml_go110.go deleted file mode 100644 index 94abc17..0000000 --- a/yaml_go110.go +++ /dev/null @@ -1,31 +0,0 @@ -// This file contains changes that are only compatible with go 1.10 and onwards. - -//go:build go1.10 -// +build go1.10 - -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yaml - -import "encoding/json" - -// DisallowUnknownFields configures the JSON decoder to error out if unknown -// fields come along, instead of dropping them by default. -func DisallowUnknownFields(d *json.Decoder) *json.Decoder { - d.DisallowUnknownFields() - return d -} diff --git a/yaml_go110_test.go b/yaml_go110_test.go deleted file mode 100644 index 629fff4..0000000 --- a/yaml_go110_test.go +++ /dev/null @@ -1,63 +0,0 @@ -//go:build go1.10 -// +build go1.10 - -/* -Copyright 2021 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package yaml - -import ( - "fmt" - "testing" -) - -func TestUnmarshalWithTags(t *testing.T) { - type WithTaggedField struct { - Field string `json:"field"` - } - - t.Run("Known tagged field", func(t *testing.T) { - y := []byte(`field: "hello"`) - v := WithTaggedField{} - if err := Unmarshal(y, &v, DisallowUnknownFields); err != nil { - t.Errorf("unexpected error: %v", err) - } - if v.Field != "hello" { - t.Errorf("v.Field=%v, want 'hello'", v.Field) - } - - }) - t.Run("With unknown tagged field", func(t *testing.T) { - y := []byte(`unknown: "hello"`) - v := WithTaggedField{} - err := Unmarshal(y, &v, DisallowUnknownFields) - if err == nil { - t.Errorf("want error because of unknown field, got : v=%#v", v) - } - }) - -} - -func exampleUnknown() { - type WithTaggedField struct { - Field string `json:"field"` - } - y := []byte(`unknown: "hello"`) - v := WithTaggedField{} - fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields)) - // Ouptut: - // unmarshaling JSON: while decoding JSON: json: unknown field "unknown" -} diff --git a/yaml_test.go b/yaml_test.go index 785162a..00fa6b8 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -17,16 +17,13 @@ limitations under the License. package yaml import ( - "encoding/json" "fmt" "math" "reflect" - "sort" "strconv" "testing" - "github.com/davecgh/go-spew/spew" - yaml "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) /* Test helper functions */ @@ -38,8 +35,10 @@ func strPtr(str string) *string { type errorType int const ( - noErrorsType errorType = 0 - fatalErrorsType errorType = 1 << iota + noErrorsType errorType = 0 + strictErrorsType errorType = 1 << iota + fatalErrorsType + strictAndFatalErrorsType errorType = strictErrorsType | fatalErrorsType ) type unmarshalTestCase struct { @@ -49,14 +48,15 @@ type unmarshalTestCase struct { err errorType } -type testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) error +type testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) (strictErrors []error, err error) var ( - funcUnmarshal testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) error { - return Unmarshal(yamlBytes, obj) + funcUnmarshal testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) (strictErrors []error, err error) { + err = Unmarshal(yamlBytes, obj) + return []error{}, err } - funcUnmarshalStrict testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) error { + funcUnmarshalStrict testUnmarshalFunc = func(yamlBytes []byte, obj interface{}) (strictErrors []error, err error) { return UnmarshalStrict(yamlBytes, obj) } ) @@ -81,13 +81,19 @@ func testUnmarshal(t *testing.T, f testUnmarshalFunc, tests map[string]unmarshal t.Errorf("unmarshalTest.ptr %#v is not a pointer to a zero value", test.decodeInto) } - err := f(test.encoded, value.Interface()) - if err != nil && test.err == noErrorsType { - t.Errorf("error unmarshaling YAML: %v", err) + strictErrors, err := f(test.encoded, value.Interface()) + if err != nil && test.err&fatalErrorsType == 0 { + t.Errorf("unexpected fatal error, unmarshaling YAML: %v", err) + } + if len(strictErrors) > 0 && test.err&strictErrorsType == 0 { + t.Errorf("unexpected strict error, unmarshaling YAML: %v", strictErrors) } if err == nil && test.err&fatalErrorsType != 0 { t.Errorf("expected a fatal error, but no fatal error was returned, yaml: `%s`", test.encoded) } + if len(strictErrors) == 0 && test.err&strictErrorsType != 0 { + t.Errorf("expected strict errors, but no strict error was returned, yaml: `%s`", test.encoded) + } if test.err&fatalErrorsType != 0 { // Don't check output if error is fatal @@ -106,19 +112,16 @@ type yamlToJSONTestcase struct { json string // By default we test that reversing the output == input. But if there is a // difference in the reversed output, you can optionally specify it here. - yaml_reverse_overwrite *string - err errorType + yamlReverseOverwrite *string + err errorType } -type testYAMLToJSONFunc = func(yamlBytes []byte) ([]byte, error) +type testYAMLToJSONFunc = func(yamlBytes []byte) (json []byte, strictErrors []error, err error) var ( - funcYAMLToJSON testYAMLToJSONFunc = func(yamlBytes []byte) ([]byte, error) { - return YAMLToJSON(yamlBytes) - } - - funcYAMLToJSONStrict testYAMLToJSONFunc = func(yamlBytes []byte) ([]byte, error) { - return YAMLToJSONStrict(yamlBytes) + funcYAMLToJSON testYAMLToJSONFunc = func(yamlBytes []byte) (json []byte, strictErrors []error, err error) { + json, err = YAMLToJSON(yamlBytes) + return json, []error{}, err } ) @@ -126,13 +129,19 @@ func testYAMLToJSON(t *testing.T, f testYAMLToJSONFunc, tests map[string]yamlToJ for testName, test := range tests { t.Run(fmt.Sprintf("%s_YAMLToJSON", testName), func(t *testing.T) { // Convert Yaml to Json - jsonBytes, err := f([]byte(test.yaml)) - if err != nil && test.err == noErrorsType { - t.Errorf("Failed to convert YAML to JSON, yaml: `%s`, err: %v", test.yaml, err) + jsonBytes, strictErrors, err := f([]byte(test.yaml)) + if err != nil && test.err&fatalErrorsType == 0 { + t.Errorf("unexpected fatal error, convert YAML to JSON, yaml: `%s`, err: %v", test.yaml, err) + } + if len(strictErrors) > 0 && test.err&strictErrorsType == 0 { + t.Errorf("unexpected strict error, convert YAML to JSON, yaml: `%s`, err: %v", test.yaml, strictErrors) } if err == nil && test.err&fatalErrorsType != 0 { t.Errorf("expected a fatal error, but no fatal error was returned, yaml: `%s`", test.yaml) } + if len(strictErrors) == 0 && test.err&strictErrorsType != 0 { + t.Errorf("expected strict errors, but no strict error was returned, yaml: `%s`", test.yaml) + } if test.err&fatalErrorsType != 0 { // Don't check output if error is fatal @@ -156,8 +165,8 @@ func testYAMLToJSON(t *testing.T, f testYAMLToJSONFunc, tests map[string]yamlToJ correctYamlString := test.yaml // If a special reverse string was specified, use that instead. - if test.yaml_reverse_overwrite != nil { - correctYamlString = *test.yaml_reverse_overwrite + if test.yamlReverseOverwrite != nil { + correctYamlString = *test.yamlReverseOverwrite } // Check it against the expected output. @@ -173,15 +182,13 @@ func testYAMLToJSON(t *testing.T, f testYAMLToJSONFunc, tests map[string]yamlToJ type MarshalTest struct { A string B int64 - // Would like to test float64, but it's not supported in go-yaml. - // (See https://github.com/go-yaml/yaml/issues/83.) - C float32 + C float64 } func TestMarshal(t *testing.T) { - f32String := strconv.FormatFloat(math.MaxFloat32, 'g', -1, 32) - s := MarshalTest{"a", math.MaxInt64, math.MaxFloat32} - e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f32String)) + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt64, math.MaxFloat64} + e := []byte(fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt64, f64String)) y, err := Marshal(s) if err != nil { @@ -194,6 +201,27 @@ func TestMarshal(t *testing.T) { } } +type MarshalTestArray struct { + A []int +} + +func TestMarshalArray(t *testing.T) { + arr := MarshalTestArray{ + A: []int{1, 2, 3}, + } + e := []byte("A:\n- 1\n- 2\n- 3\n") + + y, err := Marshal(arr) + if err != nil { + t.Errorf("error marshaling YAML: %v", err) + } + + if !reflect.DeepEqual(y, e) { + t.Errorf("marshal YAML was unsuccessful, expected: %#v, got: %#v", + string(e), string(y)) + } +} + type UnmarshalUntaggedStruct struct { A string True string @@ -248,27 +276,12 @@ type UnmarshalEmbedRecursiveStruct struct { func TestUnmarshal(t *testing.T) { tests := map[string]unmarshalTestCase{ - // casematched / non-casematched untagged keys + // casematched untagged keys "untagged casematched string key": { encoded: []byte("A: test"), decodeInto: new(UnmarshalUntaggedStruct), decoded: UnmarshalUntaggedStruct{A: "test"}, }, - "untagged non-casematched string key": { - encoded: []byte("a: test"), - decodeInto: new(UnmarshalUntaggedStruct), - decoded: UnmarshalUntaggedStruct{A: "test"}, - }, - "untagged casematched boolean key": { - encoded: []byte("True: test"), - decodeInto: new(UnmarshalUntaggedStruct), - decoded: UnmarshalUntaggedStruct{True: "test"}, - }, - "untagged non-casematched boolean key": { - encoded: []byte("true: test"), - decodeInto: new(UnmarshalUntaggedStruct), - decoded: UnmarshalUntaggedStruct{True: "test"}, - }, // casematched / non-casematched tagged keys "tagged casematched string key": { @@ -294,12 +307,12 @@ func TestUnmarshal(t *testing.T) { "tagged casematched boolean key (yes)": { encoded: []byte("Yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesUpper: "test"}, }, "tagged non-casematched boolean key (yes)": { encoded: []byte("yes: test"), decodeInto: new(UnmarshalTaggedStruct), - decoded: UnmarshalTaggedStruct{TrueLower: "test"}, + decoded: UnmarshalTaggedStruct{YesLower: "test"}, }, "tagged integer key": { encoded: []byte("3: test"), @@ -341,7 +354,7 @@ func TestUnmarshal(t *testing.T) { "boolean value (no) into string field": { encoded: []byte("a: no"), decodeInto: new(UnmarshalStruct), - decoded: UnmarshalStruct{A: "false"}, + decoded: UnmarshalStruct{A: "no"}, }, // decode into complex fields @@ -388,7 +401,7 @@ func TestUnmarshal(t *testing.T) { encoded: []byte("Yes:"), decodeInto: new(map[string]struct{}), decoded: map[string]struct{}{ - "true": {}, + "Yes": {}, }, }, "string map: decode integer key": { @@ -415,19 +428,19 @@ func TestUnmarshal(t *testing.T) { "decode 2^53 + 1 into interface": { encoded: []byte("9007199254740993"), decodeInto: new(interface{}), - decoded: 9.007199254740992e+15, + decoded: int64(9007199254740993), }, // decode into interface "float into interface": { encoded: []byte("3.0"), decodeInto: new(interface{}), - decoded: float64(3), + decoded: int64(3), }, "integer into interface": { encoded: []byte("3"), decodeInto: new(interface{}), - decoded: float64(3), + decoded: int64(3), }, "empty vs empty string into interface": { encoded: []byte("a: \"\"\nb: \n"), @@ -438,47 +451,6 @@ func TestUnmarshal(t *testing.T) { }, }, - // duplicate (non-casematched) keys (NOTE: this is very non-ideal behaviour!) - "decode duplicate (non-casematched) into nested struct 1": { - encoded: []byte("a:\n a: 1\n b: 1\n c: test\n\nA:\n a: 2"), - decodeInto: new(UnmarshalNestedStruct), - decoded: UnmarshalNestedStruct{A: UnmarshalStruct{A: "1", B: strPtr("1"), C: "test"}}, - }, - "decode duplicate (non-casematched) into nested struct 2": { - encoded: []byte("A:\n a: 1\n b: 1\n c: test\na:\n a: 2"), - decodeInto: new(UnmarshalNestedStruct), - decoded: UnmarshalNestedStruct{A: UnmarshalStruct{A: "2", B: strPtr("1"), C: "test"}}, - }, - "decode duplicate (non-casematched) into nested slice 1": { - encoded: []byte("a:\n - a: abc\n b: def\nA:\n - a: 123"), - decodeInto: new(UnmarshalSlice), - decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "abc", B: strPtr("def")}}}, - }, - "decode duplicate (non-casematched) into nested slice 2": { - encoded: []byte("A:\n - a: abc\n b: def\na:\n - a: 123"), - decodeInto: new(UnmarshalSlice), - decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "123", B: strPtr("def")}}}, - }, - "decode duplicate (non-casematched) into nested string map 1": { - encoded: []byte("a:\n b: 1\nA:\n c: 1"), - decodeInto: new(UnmarshalStringMap), - decoded: UnmarshalStringMap{map[string]string{"b": "1", "c": "1"}}, - }, - "decode duplicate (non-casematched) into nested string map 2": { - encoded: []byte("A:\n b: 1\na:\n c: 1"), - decodeInto: new(UnmarshalStringMap), - decoded: UnmarshalStringMap{map[string]string{"b": "1", "c": "1"}}, - }, - "decode duplicate (non-casematched) into string map": { - encoded: []byte("a: test\nb: test\nA: test2"), - decodeInto: new(map[string]string), - decoded: map[string]string{ - "a": "test", - "A": "test2", - "b": "test", - }, - }, - // decoding embeded structs "decode embeded struct": { encoded: []byte("a: testA\nb: testB"), @@ -510,8 +482,6 @@ func TestUnmarshal(t *testing.T) { B: "testB", }, }, - - // BUG: type info gets lost (#58) "decode embeded struct and cast integer to string": { encoded: []byte("a: 11\nb: testB"), decodeInto: new(UnmarshalEmbedStruct), @@ -521,7 +491,6 @@ func TestUnmarshal(t *testing.T) { }, B: "testB", }, - err: fatalErrorsType, }, "decode embeded structpointer and cast integer to string": { encoded: []byte("a: 11\nb: testB"), @@ -532,7 +501,6 @@ func TestUnmarshal(t *testing.T) { }, B: "testB", }, - err: fatalErrorsType, }, // decoding into incompatible type @@ -541,19 +509,7 @@ func TestUnmarshal(t *testing.T) { decodeInto: new(UnmarshalStringMap), err: fatalErrorsType, }, - } - - t.Run("Unmarshal", func(t *testing.T) { - testUnmarshal(t, funcUnmarshal, tests) - }) - t.Run("UnmarshalStrict", func(t *testing.T) { - testUnmarshal(t, funcUnmarshalStrict, tests) - }) -} - -func TestUnmarshalStrictFails(t *testing.T) { - tests := map[string]unmarshalTestCase{ // decoding with duplicate values "decode into struct pointer map with duplicate string value": { encoded: []byte("a:\n a: TestA\n b: ID-A\n b: ID-1"), @@ -561,23 +517,19 @@ func TestUnmarshalStrictFails(t *testing.T) { decoded: map[string]*UnmarshalStruct{ "a": {A: "TestA", B: strPtr("ID-1")}, }, + err: fatalErrorsType, }, "decode into string field with duplicate boolean value": { encoded: []byte("a: true\na: false"), decodeInto: new(UnmarshalStruct), decoded: UnmarshalStruct{A: "false"}, + err: fatalErrorsType, }, "decode into slice with duplicate string-boolean value": { encoded: []byte("a:\n- b: abc\n a: 32\n b: 123"), decodeInto: new(UnmarshalSlice), decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "32", B: strPtr("123")}}}, - }, - - // decoding with unknown fields - "decode into struct with unknown field": { - encoded: []byte("a: TestB\nb: ID-B\nunknown: Some-Value"), - decodeInto: new(UnmarshalStruct), - decoded: UnmarshalStruct{A: "TestB", B: strPtr("ID-B")}, + err: fatalErrorsType, }, // decoding with duplicate complex values @@ -585,16 +537,19 @@ func TestUnmarshalStrictFails(t *testing.T) { encoded: []byte("a:\n a: 1\na:\n a: 2"), decodeInto: new(UnmarshalNestedStruct), decoded: UnmarshalNestedStruct{A: UnmarshalStruct{A: "2"}}, + err: fatalErrorsType, }, "decode duplicate into nested slice": { encoded: []byte("a:\n - a: abc\n b: def\na:\n - a: 123"), decodeInto: new(UnmarshalSlice), decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "123"}}}, + err: fatalErrorsType, }, "decode duplicate into nested string map": { encoded: []byte("a:\n b: 1\na:\n c: 1"), decodeInto: new(UnmarshalStringMap), decoded: UnmarshalStringMap{map[string]string{"c": "1"}}, + err: fatalErrorsType, }, "decode duplicate into string map": { encoded: []byte("a: test\nb: test\na: test2"), @@ -603,6 +558,86 @@ func TestUnmarshalStrictFails(t *testing.T) { "a": "test2", "b": "test", }, + err: fatalErrorsType, + }, + + // duplicate (non-casematched) keys + "decode duplicate (non-casematched) into string map": { + encoded: []byte("a: test\nb: test\nA: test2"), + decodeInto: new(map[string]string), + decoded: map[string]string{ + "a": "test", + "A": "test2", + "b": "test", + }, + }, + } + + t.Run("Unmarshal", func(t *testing.T) { + testUnmarshal(t, funcUnmarshal, tests) + }) + + t.Run("UnmarshalStrict", func(t *testing.T) { + testUnmarshal(t, funcUnmarshalStrict, tests) + }) +} + +func TestUnmarshalStrictFails(t *testing.T) { + tests := map[string]unmarshalTestCase{ + // non-casematched untagged keys + "untagged non-casematched string key": { + encoded: []byte("a: test"), + decodeInto: new(UnmarshalUntaggedStruct), + decoded: UnmarshalUntaggedStruct{}, + }, + "untagged casematched boolean key": { + encoded: []byte("True: test"), + decodeInto: new(UnmarshalUntaggedStruct), + decoded: UnmarshalUntaggedStruct{}, // BUG: because True is a boolean, it is converted to the string "true" which does not match the fieldname + }, + "untagged non-casematched boolean key": { + encoded: []byte("true: test"), + decodeInto: new(UnmarshalUntaggedStruct), + decoded: UnmarshalUntaggedStruct{}, + }, + + // duplicate (non-casematched) keys -> cause unknown field errors + "decode duplicate (non-casematched) into nested struct 1": { + encoded: []byte("a:\n a: 1\n b: 1\n c: test\n\nA:\n a: 2"), + decodeInto: new(UnmarshalNestedStruct), + decoded: UnmarshalNestedStruct{A: UnmarshalStruct{A: "1", B: strPtr("1"), C: "test"}}, + }, + "decode duplicate (non-casematched) into nested struct 2": { + encoded: []byte("A:\n a: 1\n b: 1\n c: test\na:\n a: 2"), + decodeInto: new(UnmarshalNestedStruct), + decoded: UnmarshalNestedStruct{A: UnmarshalStruct{A: "2"}}, + }, + "decode duplicate (non-casematched) into nested slice 1": { + encoded: []byte("a:\n - a: abc\n b: def\nA:\n - a: 123"), + decodeInto: new(UnmarshalSlice), + decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "abc", B: strPtr("def")}}}, + }, + "decode duplicate (non-casematched) into nested slice 2": { + encoded: []byte("A:\n - a: abc\n b: def\na:\n - a: 123"), + decodeInto: new(UnmarshalSlice), + decoded: UnmarshalSlice{[]UnmarshalStruct{{A: "123"}}}, + }, + "decode duplicate (non-casematched) into nested string map 1": { + encoded: []byte("a:\n b: 1\nA:\n c: 1"), + decodeInto: new(UnmarshalStringMap), + decoded: UnmarshalStringMap{map[string]string{"b": "1"}}, + }, + "decode duplicate (non-casematched) into nested string map 2": { + encoded: []byte("A:\n b: 1\na:\n c: 1"), + decodeInto: new(UnmarshalStringMap), + decoded: UnmarshalStringMap{map[string]string{"c": "1"}}, + }, + + // decoding with unknown fields + "decode into struct with unknown field": { + encoded: []byte("a: TestB\nb: ID-B\nunknown: Some-Value"), + decodeInto: new(UnmarshalStruct), + decoded: UnmarshalStruct{A: "TestB", B: strPtr("ID-B")}, }, } @@ -613,7 +648,7 @@ func TestUnmarshalStrictFails(t *testing.T) { t.Run("UnmarshalStrict", func(t *testing.T) { failTests := map[string]unmarshalTestCase{} for name, test := range tests { - test.err = fatalErrorsType + test.err |= strictErrorsType failTests[name] = test } testUnmarshal(t, funcUnmarshalStrict, failTests) @@ -631,63 +666,63 @@ func TestYAMLToJSON(t *testing.T) { json: `{"t":null}`, }, "boolean value": { - yaml: "t: True\n", - json: `{"t":true}`, - yaml_reverse_overwrite: strPtr("t: true\n"), + yaml: "t: True\n", + json: `{"t":true}`, + yamlReverseOverwrite: strPtr("t: true\n"), }, "boolean value (no)": { - yaml: "t: no\n", - json: `{"t":false}`, - yaml_reverse_overwrite: strPtr("t: false\n"), + yaml: "t: no\n", + json: `{"t":"no"}`, + yamlReverseOverwrite: strPtr("t: \"no\"\n"), }, "integer value (2^53 + 1)": { - yaml: "t: 9007199254740993\n", - json: `{"t":9007199254740993}`, - yaml_reverse_overwrite: strPtr("t: 9007199254740993\n"), + yaml: "t: 9007199254740993\n", + json: `{"t":9007199254740993}`, + yamlReverseOverwrite: strPtr("t: 9007199254740993\n"), }, "integer value (1000000000000000000000000000000000000)": { - yaml: "t: 1000000000000000000000000000000000000\n", - json: `{"t":1e+36}`, - yaml_reverse_overwrite: strPtr("t: 1e+36\n"), + yaml: "t: 1000000000000000000000000000000000000\n", + json: `{"t":1e+36}`, + yamlReverseOverwrite: strPtr("t: 1e+36\n"), }, "line-wrapped string value": { yaml: "t: this is very long line with spaces and it must be longer than 80 so we will repeat\n that it must be longer that 80\n", json: `{"t":"this is very long line with spaces and it must be longer than 80 so we will repeat that it must be longer that 80"}`, }, "empty yaml value": { - yaml: "t: ", - json: `{"t":null}`, - yaml_reverse_overwrite: strPtr("t: null\n"), + yaml: "t: ", + json: `{"t":null}`, + yamlReverseOverwrite: strPtr("t: null\n"), }, "boolean key": { - yaml: "True: a", - json: `{"true":"a"}`, - yaml_reverse_overwrite: strPtr("\"true\": a\n"), + yaml: "True: a", + json: `{"true":"a"}`, + yamlReverseOverwrite: strPtr("\"true\": a\n"), }, "boolean key (no)": { - yaml: "no: a", - json: `{"false":"a"}`, - yaml_reverse_overwrite: strPtr("\"false\": a\n"), + yaml: "no: a", + json: `{"no":"a"}`, + yamlReverseOverwrite: strPtr("\"no\": a\n"), }, "integer key": { - yaml: "1: a", - json: `{"1":"a"}`, - yaml_reverse_overwrite: strPtr("\"1\": a\n"), + yaml: "1: a", + json: `{"1":"a"}`, + yamlReverseOverwrite: strPtr("\"1\": a\n"), }, "float key": { - yaml: "1.2: a", - json: `{"1.2":"a"}`, - yaml_reverse_overwrite: strPtr("\"1.2\": a\n"), + yaml: "1.2: a", + json: `{"1.2":"a"}`, + yamlReverseOverwrite: strPtr("\"1.2\": a\n"), }, "large integer key": { - yaml: "1000000000000000000000000000000000000: a", - json: `{"1e+36":"a"}`, - yaml_reverse_overwrite: strPtr("\"1e+36\": a\n"), + yaml: "1000000000000000000000000000000000000: a", + json: `{"1e+36":"a"}`, + yamlReverseOverwrite: strPtr("\"1e+36\": a\n"), }, "large integer key (scientific notation)": { - yaml: "1e+36: a", - json: `{"1e+36":"a"}`, - yaml_reverse_overwrite: strPtr("\"1e+36\": a\n"), + yaml: "1e+36: a", + json: `{"1e+36":"a"}`, + yamlReverseOverwrite: strPtr("\"1e+36\": a\n"), }, "string key (large integer as string)": { yaml: "\"1e+36\": a\n", @@ -706,187 +741,67 @@ func TestYAMLToJSON(t *testing.T) { json: `[{"t":"a"},{"t":{"b":1,"c":2}}]`, }, "nested struct array (json notation)": { - yaml: `[{t: a}, {t: {b: 1, c: 2}}]`, - json: `[{"t":"a"},{"t":{"b":1,"c":2}}]`, - yaml_reverse_overwrite: strPtr("- t: a\n- t:\n b: 1\n c: 2\n"), + yaml: `[{t: a}, {t: {b: 1, c: 2}}]`, + json: `[{"t":"a"},{"t":{"b":1,"c":2}}]`, + yamlReverseOverwrite: strPtr("- t: a\n- t:\n b: 1\n c: 2\n"), }, "empty struct value": { - yaml: "- t: ", - json: `[{"t":null}]`, - yaml_reverse_overwrite: strPtr("- t: null\n"), + yaml: "- t: ", + json: `[{"t":null}]`, + yamlReverseOverwrite: strPtr("- t: null\n"), }, "null struct value": { yaml: "- t: null\n", json: `[{"t":null}]`, }, "binary data": { - yaml: "a: !!binary gIGC", - json: `{"a":"\ufffd\ufffd\ufffd"}`, - yaml_reverse_overwrite: strPtr("a: \ufffd\ufffd\ufffd\n"), + yaml: "a: !!binary gIGC", + json: `{"a":"\ufffd\ufffd\ufffd"}`, + yamlReverseOverwrite: strPtr("a: \ufffd\ufffd\ufffd\n"), }, // Cases that should produce errors. "~ key": { - yaml: "~: a", - json: `{"null":"a"}`, - yaml_reverse_overwrite: strPtr("\"null\": a\n"), - err: fatalErrorsType, + yaml: "~: a", + json: `{"null":"a"}`, + yamlReverseOverwrite: strPtr("\"null\": a\n"), + err: fatalErrorsType, }, "null key": { - yaml: "null: a", - json: `{"null":"a"}`, - yaml_reverse_overwrite: strPtr("\"null\": a\n"), - err: fatalErrorsType, + yaml: "null: a", + json: `{"null":"a"}`, + yamlReverseOverwrite: strPtr("\"null\": a\n"), + err: fatalErrorsType, }, - } - t.Run("YAMLToJSON", func(t *testing.T) { - testYAMLToJSON(t, funcYAMLToJSON, tests) - }) - - t.Run("YAMLToJSONStrict", func(t *testing.T) { - testYAMLToJSON(t, funcYAMLToJSONStrict, tests) - }) -} - -func TestYAMLToJSONStrictFails(t *testing.T) { - tests := map[string]yamlToJSONTestcase{ - // expect YAMLtoJSON to pass on duplicate field names + // expect YAMLtoJSON to fail on duplicate field names "duplicate struct value": { - yaml: "foo: bar\nfoo: baz\n", - json: `{"foo":"baz"}`, - yaml_reverse_overwrite: strPtr("foo: baz\n"), + yaml: "foo: bar\nfoo: baz\n", + json: `{"foo":"baz"}`, + yamlReverseOverwrite: strPtr("foo: baz\n"), + err: fatalErrorsType, }, } t.Run("YAMLToJSON", func(t *testing.T) { testYAMLToJSON(t, funcYAMLToJSON, tests) }) - - t.Run("YAMLToJSONStrict", func(t *testing.T) { - failTests := map[string]yamlToJSONTestcase{} - for name, test := range tests { - test.err = fatalErrorsType - failTests[name] = test - } - testYAMLToJSON(t, funcYAMLToJSONStrict, failTests) - }) } -func TestJSONObjectToYAMLObject(t *testing.T) { - const bigUint64 = ((uint64(1) << 63) + 500) / 1000 * 1000 - intOrInt64 := func(i64 int64) interface{} { - if i := int(i64); i64 == int64(i) { - return i - } - return i64 - } +func TestYamlNode(t *testing.T) { + var yamlNode yaml.Node + data := []byte("a:\n b: test\n c:\n - t:\n") - tests := []struct { - name string - input map[string]interface{} - expected yaml.MapSlice - }{ - {name: "nil", expected: yaml.MapSlice(nil)}, - {name: "empty", input: map[string]interface{}{}, expected: yaml.MapSlice(nil)}, - { - name: "values", - input: map[string]interface{}{ - "nil slice": []interface{}(nil), - "nil map": map[string]interface{}(nil), - "empty slice": []interface{}{}, - "empty map": map[string]interface{}{}, - "bool": true, - "float64": float64(42.1), - "fractionless": float64(42), - "int": int(42), - "int64": int64(42), - "int64 big": float64(math.Pow(2, 62)), - "negative int64 big": -float64(math.Pow(2, 62)), - "map": map[string]interface{}{"foo": "bar"}, - "slice": []interface{}{"foo", "bar"}, - "string": string("foo"), - "uint64 big": bigUint64, - }, - expected: yaml.MapSlice{ - {Key: "nil slice"}, - {Key: "nil map"}, - {Key: "empty slice", Value: []interface{}{}}, - {Key: "empty map", Value: yaml.MapSlice(nil)}, - {Key: "bool", Value: true}, - {Key: "float64", Value: float64(42.1)}, - {Key: "fractionless", Value: int(42)}, - {Key: "int", Value: int(42)}, - {Key: "int64", Value: int(42)}, - {Key: "int64 big", Value: intOrInt64(int64(1) << 62)}, - {Key: "negative int64 big", Value: intOrInt64(-(1 << 62))}, - {Key: "map", Value: yaml.MapSlice{{Key: "foo", Value: "bar"}}}, - {Key: "slice", Value: []interface{}{"foo", "bar"}}, - {Key: "string", Value: string("foo")}, - {Key: "uint64 big", Value: bigUint64}, - }, - }, + if err := yaml.Unmarshal(data, &yamlNode); err != nil { + t.Error(err) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := JSONObjectToYAMLObject(tt.input) - sortMapSlicesInPlace(tt.expected) - sortMapSlicesInPlace(got) - if !reflect.DeepEqual(got, tt.expected) { - t.Errorf("jsonToYAML() = %v, want %v", spew.Sdump(got), spew.Sdump(tt.expected)) - } - jsonBytes, err := json.Marshal(tt.input) - if err != nil { - t.Fatalf("unexpected json.Marshal error: %v", err) - } - var gotByRoundtrip yaml.MapSlice - if err := yaml.Unmarshal(jsonBytes, &gotByRoundtrip); err != nil { - t.Fatalf("unexpected yaml.Unmarshal error: %v", err) - } - - // yaml.Unmarshal loses precision, it's rounding to the 4th last digit. - // Replicate this here in the test, but don't change the type. - for i := range got { - switch got[i].Key { - case "int64 big", "uint64 big", "negative int64 big": - switch v := got[i].Value.(type) { - case int64: - d := int64(500) - if v < 0 { - d = -500 - } - got[i].Value = int64((v+d)/1000) * 1000 - case uint64: - got[i].Value = uint64((v+500)/1000) * 1000 - case int: - d := int(500) - if v < 0 { - d = -500 - } - got[i].Value = int((v+d)/1000) * 1000 - default: - t.Fatalf("unexpected type for key %s: %v:%T", got[i].Key, v, v) - } - } - } - - if !reflect.DeepEqual(got, gotByRoundtrip) { - t.Errorf("yaml.Unmarshal(json.Marshal(tt.input)) = %v, want %v\njson: %s", spew.Sdump(gotByRoundtrip), spew.Sdump(got), string(jsonBytes)) - } - }) + dataOut, err := yaml.Marshal(&yamlNode) + if err != nil { + t.Error(err) } -} -func sortMapSlicesInPlace(x interface{}) { - switch x := x.(type) { - case []interface{}: - for i := range x { - sortMapSlicesInPlace(x[i]) - } - case yaml.MapSlice: - sort.Slice(x, func(a, b int) bool { - return x[a].Key.(string) < x[b].Key.(string) - }) + if string(data) != string(dataOut) { + t.Errorf("yaml.Node roudtrip failed: expected yaml `%s`, got `%s`", string(data), string(dataOut)) } }