diff --git a/content/content_type.go b/content/content_type.go new file mode 100644 index 0000000..d947772 --- /dev/null +++ b/content/content_type.go @@ -0,0 +1,54 @@ +package content + +import ( + "fmt" + + "github.com/luxas/deklarative/content/metadata" +) + +var _ fmt.Stringer = ContentType("") + +// ContentType specifies the content type of some stream. +// Ideally, a standard MIME notation like "application/json" shall be used. +type ContentType string //nolint:revive + +const ( + // TODO: Maybe consider moving these to their respective package? + ContentTypeYAML ContentType = "application/yaml" + ContentTypeJSON ContentType = "application/json" +) + +func (ct ContentType) ContentType() ContentType { return ct } +func (ct ContentType) String() string { return string(ct) } + +type ContentTypes []ContentType //nolint:revive + +func (cts ContentTypes) Has(want ContentType) bool { + for _, ct := range cts { + if ct == want { + return true + } + } + return false +} + +func WithContentType(ct ContentType) metadata.HeaderOption { + return metadata.SetOption(metadata.ContentTypeKey, ct.String()) +} + +/*type ContentTypeMetadataRecognizer interface { + metadata.HeaderOption +}*/ + +// ContentTyped is an interface that contains and/or supports one content type. +//nolint:revive +type ContentTyped interface { + ContentType() ContentType +} + +// ContentTypeSupporter supports potentially multiple content types. +//nolint:revive +type ContentTypeSupporter interface { + // Order _might_ carry a meaning + SupportedContentTypes() ContentTypes +} diff --git a/content/decode.go b/content/decode.go new file mode 100644 index 0000000..3771eac --- /dev/null +++ b/content/decode.go @@ -0,0 +1,300 @@ +package content + +import "io" + +type FrameDecoder interface { + DecodeFrame() (Frame, error) + + //SupportsContentType(ct ContentType) bool +} + +type Decoder interface { + Decode(into interface{}) error +} + +/*type RecognizingDecoder interface { + RecognizesData() bool +}*/ + +type ReadAllFrameDecoderCreator struct { + ContentType ContentType +} + +func (c ReadAllFrameDecoderCreator) NewFrameDecoder(r io.Reader) FrameDecoder { + return ReadAllFrameDecoder(r, c.ContentType) +} + +func (ReadAllFrameDecoderCreator) SupportsContentType(ct ContentType) bool { return true } + +func ReadAllFrameDecoder(r io.Reader, ct ContentType) FrameDecoder { + return &readAllFrameDecoder{r, ct, false} +} + +type readAllFrameDecoder struct { + r io.Reader + ct ContentType + hasBeenRead bool +} + +func (r *readAllFrameDecoder) DecodeFrame() (Frame, error) { + if r.hasBeenRead { + return nil, io.EOF + } + r.hasBeenRead = true + + data, err := io.ReadAll(r.r) + if err != nil { + return nil, err + } + + return NewFrame(r.ct, data, nil, len(data) == 0), nil +} + +/* +type DecoderOption interface { + ApplyToDecoder(target DecoderOptionTarget) +}*/ + +type OptionDisallowUnknownFields interface { + // ApplyDisallowUnknownFields applies the DisallowUnknownFields option. + // If DisallowUnknownFields == true, only known fields will be decoded, and if there are + // unknown fields in the serialized bytes, the decoder will error. For + // DisallowUnknownFields == false, the decoder will silently ignore unknown fields. + ApplyDisallowUnknownFields(DisallowUnknownFields bool) +} + +type OptionDisallowUnknownFieldsGetter interface { + // GetDisallowUnknownFields gets whether the implementer disallows unknown fields or not. + // + // Decoders defaulting this to false, but allows overriding: + // + // encoding/json + // json-iterator + // yaml.v3 + // kyaml + // + // Decoders defaulting this to false, without an option to override: + // + // k8s.io/apimachinery/pkg/util/json + // + // Decoders defaulting this to false, but allows overriding to the same value as + // the value of GetDisallowDuplicateFields() (i.e. either strict or non-strict): + // + // yaml.v2 + // sigs.k8s.io/yaml + // k8s.io/apimachinery/pkg/runtime/serializer/json + // + GetDisallowUnknownFields() bool +} + +// TODO: Maybe make a OptionDisallowDuplicateFields type + +type OptionDisallowDuplicateFieldsGetter interface { + // GetDisallowUnknownFields gets whether the implementer disallows duplicate fields or not. + // + // Decoders hardcoding this to false (i.e. duplicate fields are allowed): + // + // encoding/json + // k8s.io/apimachinery/pkg/util/json + // + // Decoders hardcoding this to true: + // + // yaml.v3 + // kyaml + // json-iterator + // + // Decoders defaulting this to false, but allows overriding to the same value as + // the value of GetDisallowUnknownFields() (i.e. either strict or non-strict): + // + // yaml.v2 + // sigs.k8s.io/yaml + // k8s.io/apimachinery/pkg/runtime/serializer/json + // + GetDisallowDuplicateFields() bool +} + +type OptionCaseSensitiveGetter interface { + // OptionCaseSensitiveGetter gets whether the implementer is case-sensitive or not. + // + // Decoders hardcoding this to false (i.e. are in-case-sensitive): + // + // encoding/json + // k8s.io/apimachinery/pkg/util/json + // sigs.k8s.io/yaml + // + // Decoders defaulting to false, but allows configuration: + // + // json-iterator + // + // Decoders hardcoding this to true: + // + // yaml.v2 + // yaml.v3 + // kyaml + // k8s.io/apimachinery/pkg/runtime/serializer/json + // + GetCaseSensitive() bool +} + +type OptionDefaultFieldNamingGetter interface { + // GetDefaultFieldNaming returns what the default naming policy is for fields of typed + // structs if no field tag is set. + // + // Decoders and encoders defaulting this to NamingConventionLowercase: + // + // yaml.v2 + // yaml.v3 + // kyaml + // + // Decoders and encoders defaulting this to NamingConventionFieldName: + // + // encoding/json + // json-iter? + // k8s.io/apimachinery/pkg/runtime/serializer/json + // k8s.io/apimachinery/pkg/util/json + // sigs.k8s.io/yaml + // + GetDefaultFieldNaming() NamingConvention +} + +type NamingConvention int + +const ( + NamingConventionLowercase NamingConvention = iota + NamingConventionFieldName +) + +/*type OptionUseNumber interface { + ApplyUseNumber(useNumber bool) +} + +type OptionUseNumberGetter interface { + GetUseNumber() bool +} + +type OptionConvertUnstructuredNumbers interface { + ApplyConvertUnstructuredNumbers(convert bool) +} + +type OptionConvertUnstructuredNumbersGetter interface { + GetConvertUnstructuredNumbers(convert bool) +}*/ + +type OptionUnknownNumberStrategy interface { + // ApplyUnknownNumberStrategy applies the given UnknownNumberStrategy option. + // If strategy is UnknownNumberStrategyAlwaysFloat64, any number without a given + // Go number type will be assigned float64. This can, however, lead to loss of + // precision, as not every integer can be expressed as a float64. Hence, there is + // UnknownNumberStrategyInt64OrFloat64, which first tries to parse the number as + // an int64, and only if that fails, assign it to a float64. This leads to that + // integers can always be round-tripped losslessly. The third option, + // UnknownNumberStrategyJSONNumber leaves the value as a string, with Int64() and + // Float64() methods (see encoding/json.Number). + ApplyUnknownNumberStrategy(strategy UnknownNumberStrategy) +} + +type OptionUnknownNumberStrategyGetter interface { + // GetUnknownNumberStrategy returns what unknown number strategy is used. + // + // Decoders that default to UnknownNumberStrategyAlwaysFloat64, but allow + // use of the json.Number method: + // + // encoding/json + // json-iterator + // + // Decoders that default to UnknownNumberStrategyAlwaysFloat64, but do not + // allow decoding into json.Number: + // + // sigs.k8s.io/yaml + // + // Decoders that default to UnknownNumberStrategyInt64OrFloat64, and do not + // allow configuring: + // + // yaml.v2 + // yaml.v3 + // kyaml + // k8s.io/apimachinery/pkg/util/json + // k8s.io/apimachinery/pkg/runtime/serializer/json + // + GetUnknownNumberStrategy() UnknownNumberStrategy +} + +type UnknownNumberStrategy int + +const ( + UnknownNumberStrategyAlwaysFloat64 UnknownNumberStrategy = 1 + iota + UnknownNumberStrategyInt64OrFloat64 + UnknownNumberStrategyJSONNumber +) + +func (s UnknownNumberStrategy) String() string { + switch s { + case UnknownNumberStrategyAlwaysFloat64: + return "AlwaysFloat64" + case UnknownNumberStrategyInt64OrFloat64: + return "Int64OrFloat64" + case UnknownNumberStrategyJSONNumber: + return "JSONNumber" + default: + return "" + } +} + +func ValidUnknownNumberStrategy(strategy UnknownNumberStrategy) bool { + return UnknownNumberStrategyAlwaysFloat64 <= strategy && + strategy <= UnknownNumberStrategyJSONNumber +} + +type DuplicateFieldsPolicy int + +const ( + DuplicateFieldsPolicyIgnore DuplicateFieldsPolicy = 1 + iota + DuplicateFieldsPolicyError + // DuplicateFieldsPolicyWarn +) + +func (p DuplicateFieldsPolicy) String() string { + switch p { + case DuplicateFieldsPolicyIgnore: + return "Ignore" + case DuplicateFieldsPolicyError: + return "Error" + default: + return "" + } +} + +func ValidDuplicateFieldsPolicy(policy DuplicateFieldsPolicy) bool { + return DuplicateFieldsPolicyIgnore <= policy && + policy <= DuplicateFieldsPolicyError +} + +type UnknownFieldsPolicy int + +const ( + UnknownFieldsPolicyIgnore UnknownFieldsPolicy = 1 + iota + UnknownFieldsPolicyError + // UnknownFieldsPolicyWarn +) + +func (p UnknownFieldsPolicy) String() string { + switch p { + case UnknownFieldsPolicyIgnore: + return "Ignore" + case UnknownFieldsPolicyError: + return "Error" + default: + return "" + } +} + +func ValidUnknownFieldsPolicy(policy UnknownFieldsPolicy) bool { + return UnknownFieldsPolicyIgnore <= policy && + policy <= UnknownFieldsPolicyError +} + +/*type DecoderOptionTarget interface { + UseNumber(bool) + AllowNonStringKeys(bool) // TODO: Figure this out + CaseSensitive(bool) // Always true? +}*/ diff --git a/content/encode.go b/content/encode.go new file mode 100644 index 0000000..0073a1a --- /dev/null +++ b/content/encode.go @@ -0,0 +1,50 @@ +package content + +import "strings" + +type Encoder interface { + Encode(obj interface{}) error +} + +type EncoderOption interface { + ApplyToEncoder(target EncoderOptionTarget) +} + +type EncoderOptionTarget interface { + SetIndent(string) // Only configurable for JSON + SortMapKeys(bool) // + LowercaseEverythingLikeYAMLv3(bool) // Or just print the field name as-is + SeqIndentStyle(string) // YAML-specific +} + +// TODO: Copy over godoc from yaml.v3 +type IsZeroer interface { + IsZero() bool +} + +type ZeroEncodePolicy int + +const ( + CheckIsZero ZeroEncodePolicy = 1 << iota + CheckIsZeroPointer + CheckIsZeroStructRecursive +) +const checkIsZeroInclusiveUpperbound = CheckIsZeroStructRecursive<<1 - 1 + +func (p ZeroEncodePolicy) String() string { + modes := []string{} + if p&CheckIsZero != 0 { + modes = append(modes, "Plain") + } + if p&CheckIsZeroPointer != 0 { + modes = append(modes, "Pointer") + } + if p&CheckIsZeroStructRecursive != 0 { + modes = append(modes, "StructRecursive") + } + return strings.Join(modes, ",") +} + +func IsValidZeroEncodePolicy(p ZeroEncodePolicy) bool { + return CheckIsZero <= p && p <= checkIsZeroInclusiveUpperbound +} diff --git a/content/errors.go b/content/errors.go new file mode 100644 index 0000000..1e06e7c --- /dev/null +++ b/content/errors.go @@ -0,0 +1,71 @@ +package content + +import ( + "fmt" + + "github.com/luxas/deklarative/content/structerr" +) + +// RecognizeError is returned from NewRecognizingReader and +// NewRecognizingWriter if recognition failed. +type RecognizeError struct { + Metadata Metadata + Peek []byte + Underlying error +} + +// RecognizeError may carry an underlying error cause. +var _ structerr.UnwrapError = &RecognizeError{} + +func (e *RecognizeError) Error() string { + msg := "couldn't recognize content type" + if e.Underlying != nil { + msg = fmt.Sprintf("%s: %v", msg, e.Underlying) + } + return msg +} + +func (e *RecognizeError) Is(target error) bool { + //nolint:errorlint + _, ok := target.(*RecognizeError) + return ok +} + +func (e *RecognizeError) Unwrap() error { return e.Underlying } + +// Enforce all struct errors implementing structerr.StructError. +var _ structerr.StructError = &UnsupportedContentTypeError{} + +// ErrUnsupportedContentType creates a new *UnsupportedContentTypeError. +func ErrUnsupportedContentType(unsupported ContentType, supported ...ContentType) *UnsupportedContentTypeError { + return &UnsupportedContentTypeError{Unsupported: unsupported, Supported: supported} +} + +// UnsupportedContentTypeError describes that the supplied content type is not supported by an +// implementation handling different content types. +// +// This error can be checked for equality using errors.Is(err, &UnsupportedContentTypeError{}). +type UnsupportedContentTypeError struct { + // Unsupported is the content type that was given but not supported + // +required + Unsupported ContentType + // Supported is optional; if len(Supported) != 0, it lists the content types that indeed + // are supported by the implementation. If len(Supported) == 0, it should not be used + // as an indicator. + // +optional + Supported []ContentType +} + +func (e *UnsupportedContentTypeError) Error() string { + msg := fmt.Sprintf("unsupported content type: %q", e.Unsupported) + if len(e.Supported) != 0 { + msg = fmt.Sprintf("%s. supported content types: %v", msg, e.Supported) + } + return msg +} + +func (e *UnsupportedContentTypeError) Is(target error) bool { + //nolint:errorlint + _, ok := target.(*UnsupportedContentTypeError) + return ok +} diff --git a/content/frame.go b/content/frame.go new file mode 100644 index 0000000..d3a0f70 --- /dev/null +++ b/content/frame.go @@ -0,0 +1,27 @@ +package content + +type Frame interface { + // TODO: Is there a need for a deep-copy function? + + ContentType() ContentType + Content() []byte + + DecodedGeneric() interface{} + IsEmpty() bool +} + +func NewFrame(ct ContentType, content []byte, obj interface{}, isEmpty bool) Frame { + return &frame{ct, content, obj, isEmpty} +} + +type frame struct { + ct ContentType + content []byte + obj interface{} + isEmpty bool +} + +func (f *frame) ContentType() ContentType { return f.ct } +func (f *frame) Content() []byte { return f.content } +func (f *frame) DecodedGeneric() interface{} { return f.obj } +func (f *frame) IsEmpty() bool { return f.isEmpty } diff --git a/content/go.mod b/content/go.mod new file mode 100644 index 0000000..56df01f --- /dev/null +++ b/content/go.mod @@ -0,0 +1,5 @@ +module github.com/luxas/deklarative/content + +go 1.16 + +require github.com/stretchr/testify v1.7.0 diff --git a/content/go.sum b/content/go.sum new file mode 100644 index 0000000..26500d5 --- /dev/null +++ b/content/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/content/metadata.go b/content/metadata.go new file mode 100644 index 0000000..b15aa91 --- /dev/null +++ b/content/metadata.go @@ -0,0 +1,100 @@ +package content + +import ( + "encoding/json" + "net/textproto" + "net/url" + + "github.com/luxas/deklarative/content/metadata" +) + +// Metadata is the interface that's common to contentMetadataOptions and a wrapper +// around a HTTP request. +type Metadata interface { + metadata.Header + metadata.HeaderOption + + // Apply applies the given Options to itself and returns itself, without + // any deep-copying. + Apply(opts ...metadata.HeaderOption) Metadata + // ContentLength retrieves the standard "Content-Length" header + ContentLength() (int64, bool) + // ContentType retrieves the standard "Content-Type" header + ContentType() (ContentType, bool) + // ContentLocation retrieves the custom "X-Content-Location" header + ContentLocation() (*url.URL, bool) + + // Clone makes a deep copy of the Metadata + // TODO: Do we need this anymore? + Clone() Metadata + + ToContainer() MetadataContainer +} + +var _ Metadata = &contentMetadata{} +var _ json.Marshaler = &contentMetadata{} + +func (m *contentMetadata) MarshalJSON() ([]byte, error) { + return json.Marshal(m.MIMEHeader) +} + +func (m *contentMetadata) ApplyToHeader(target metadata.Header) { + for k, vals := range m.MIMEHeader { + for i, val := range vals { + if i == 0 { + target.Set(k, val) + } else { + target.Add(k, val) + } + } + } +} + +func (m *contentMetadata) Apply(opts ...metadata.HeaderOption) Metadata { + for _, opt := range opts { + opt.ApplyToHeader(m) + } + return m +} + +func (m *contentMetadata) ContentLength() (int64, bool) { + return metadata.GetInt64(m, metadata.ContentLengthKey) +} + +func (m *contentMetadata) ContentType() (ContentType, bool) { + ct, ok := metadata.GetString(m, metadata.ContentTypeKey) + return ContentType(ct), ok +} + +func (m *contentMetadata) ContentLocation() (*url.URL, bool) { + return metadata.GetURL(m, metadata.XContentLocationKey) +} + +func (m *contentMetadata) ToContainer() MetadataContainer { + return &metadataContainer{m} +} + +func (m *contentMetadata) Clone() Metadata { + m2 := make(textproto.MIMEHeader, len(m.MIMEHeader)) + for k, v := range m.MIMEHeader { + m2[k] = v + } + return &contentMetadata{m2} +} + +type MetadataContainer interface { + // ContentMetadata + ContentMetadata() Metadata +} + +func NewMetadata(opts ...metadata.HeaderOption) Metadata { + return (&contentMetadata{MIMEHeader: textproto.MIMEHeader{}}).Apply(opts...) +} + +type contentMetadata struct { + textproto.MIMEHeader +} + +type metadataContainer struct{ m Metadata } + +func (b *metadataContainer) ContentMetadata() Metadata { return b.m } diff --git a/content/metadata/metadata.go b/content/metadata/metadata.go new file mode 100644 index 0000000..6379d65 --- /dev/null +++ b/content/metadata/metadata.go @@ -0,0 +1,155 @@ +// Metadata contains an interface to work with HTTP-like headers carrying metadata about +// some stream. +package metadata + +import ( + "mime" + "net/textproto" + "net/url" + "strconv" + "strings" +) + +/* + Metadata origin in the system by default: + + stream.FromFile -> stream.Reader + - X-Content-Location + - Content-Length + + stream.FromBytes -> stream.Reader + - Content-Length + + stream.FromString -> stream.Reader + - Content-Length + + stream.ToFile -> stream.Writer + - X-Content-Location + + stream.ToBuffer -> stream.Writer + + frame.NewYAMLReader -> frame.Reader + - Content-Type => YAML + + frame.NewJSONReader -> frame.Reader + - Content-Type => JSON + + frame.newRecognizingReader -> frame.Reader + - If Content-Type is set, use that ContentType + - If X-Content-Location is set, try deduce ContentType from that + - Peek the stream, and try to deduce the ContentType from that + +*/ + +// "Known" headers to the system by default, but any other header can also be attached. +const ( + XContentLocationKey = "X-Content-Location" + + ContentLengthKey = "Content-Length" + ContentTypeKey = "Content-Type" + AcceptKey = "Accept" + + // TODO: Add Content-Encoding and Last-Modified? +) + +type HeaderOption interface { + // TODO: Rename to ApplyMetadataHeader? + ApplyToHeader(target Header) +} + +var _ HeaderOption = setHeaderOption{} + +func SetOption(k, v string) HeaderOption { + return setHeaderOption{Key: k, Value: v} +} + +func WithContentLength(length int64) HeaderOption { + return SetOption(ContentLengthKey, strconv.FormatInt(length, 10)) +} + +func WithContentLocation(loc string) HeaderOption { + return SetOption(XContentLocationKey, loc) +} + +func WithAccept(accepts ...string) HeaderOption { + return addHeaderOption{Key: AcceptKey, Values: accepts} +} + +type setHeaderOption struct{ Key, Value string } + +func (o setHeaderOption) ApplyToHeader(target Header) { + target.Set(o.Key, o.Value) +} + +type addHeaderOption struct { + Key string + Values []string +} + +func (o addHeaderOption) ApplyToHeader(target Header) { + for _, val := range o.Values { + target.Add(o.Key, val) + } +} + +// Make sure the interface is compatible with the targeted textproto.MIMEHeader. +var _ Header = textproto.MIMEHeader{} + +// Express the string-string map interface of the net/textproto.Header map. +type Header interface { + Add(key, value string) + Set(key, value string) + Get(key string) string + Values(key string) []string + Del(key string) +} + +// TODO: Public or private? + +func GetString(m Header, key string) (string, bool) { + if len(m.Values(key)) == 0 { + return "", false + } + return m.Get(key), true +} + +func GetBytes(m Header, key string) ([]byte, bool) { + str, ok := GetString(m, key) + if !ok { + return nil, false + } + return []byte(str), true +} + +func GetInt64(m Header, key string) (int64, bool) { + i, err := strconv.ParseInt(m.Get(key), 10, 64) + if err != nil { + return 0, false + } + return i, true +} + +func GetURL(m Header, key string) (*url.URL, bool) { + str, ok := GetString(m, key) + if !ok { + return nil, false + } + u, err := url.Parse(str) + if err != nil { + return nil, false + } + return u, true +} + +func GetMediaTypes(m Header, key string) (mediaTypes []string, err error) { + for _, commaSepVal := range m.Values(key) { + for _, mediaTypeStr := range strings.Split(commaSepVal, ",") { + mediaType, _, err := mime.ParseMediaType(mediaTypeStr) + if err != nil { + return nil, err + } + mediaTypes = append(mediaTypes, mediaType) + } + } + return +} diff --git a/content/metadata/metadata_test.go b/content/metadata/metadata_test.go new file mode 100644 index 0000000..ea09482 --- /dev/null +++ b/content/metadata/metadata_test.go @@ -0,0 +1,49 @@ +package metadata + +import ( + "net/textproto" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetMediaTypes(t *testing.T) { + tests := []struct { + name string + opts []HeaderOption + key string + wantMediaTypes []string + wantErr error + }{ + { + name: "multiple keys, and values in one key", + opts: []HeaderOption{ + WithAccept("application/yaml", "application/xml"), + WithAccept("application/json"), + WithAccept("text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8"), + }, + key: AcceptKey, + wantMediaTypes: []string{ + "application/yaml", + "application/xml", + "application/json", + "text/html", + "application/xhtml+xml", + "application/xml", + "image/webp", + "*/*", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := textproto.MIMEHeader{} + for _, opt := range tt.opts { + opt.ApplyToHeader(h) + } + gotMediaTypes, err := GetMediaTypes(h, tt.key) + assert.Equal(t, tt.wantMediaTypes, gotMediaTypes) + assert.ErrorIs(t, err, tt.wantErr) + }) + } +} diff --git a/content/recognizing.go b/content/recognizing.go new file mode 100644 index 0000000..5a6e2cb --- /dev/null +++ b/content/recognizing.go @@ -0,0 +1,74 @@ +package content + +import ( + "path/filepath" + + "github.com/luxas/deklarative/content/metadata" +) + +type MetadataRecognizer interface { + FromMetadata(h metadata.Header) ContentType + + //metadata.HeaderOption + + // SupportedContentTypes() tells about what ContentTypes are supported by this recognizer + //ContentTypeSupporter +} + +type PeekRecognizer interface { + FromPeekBytes(peek []byte) ContentType + + // SupportedContentTypes() tells about what ContentTypes are supported by this recognizer + //ContentTypeSupporter +} + +var _ MetadataRecognizer = ExtToContentTypeMap{} + +// ExtToContentTypeMap is a metadata.HeaderOption implementation that +// based on the X-Content-Location header sets the content type. The +// map maps the extension to the content type. +type ExtToContentTypeMap map[string]ContentType + +func (m ExtToContentTypeMap) FromMetadata(h metadata.Header) ContentType { + loc, ok := metadata.GetString(h, metadata.XContentLocationKey) + if !ok { + return "" + } + ext := filepath.Ext(loc) + ct, ok := m[ext] + if !ok { + return "" + } + + return ct +} + +func HeaderOptionFromMetaRecognizer(rec MetadataRecognizer) metadata.HeaderOption { + return &metadataRecognizerOption{rec} +} + +type metadataRecognizerOption struct{ MetadataRecognizer } + +func (o *metadataRecognizerOption) ApplyToHeader(h metadata.Header) { + ct, _ := metadata.GetString(h, metadata.ContentTypeKey) + if len(ct) != 0 { + return // no need to recognize + } + + ct = string(o.FromMetadata(h)) + if len(ct) == 0 { + return // no recognition result + } + + h.Set(metadata.ContentTypeKey, ct) +} + +/*func (m ExtToContentTypeMap) SupportedContentTypes() ContentTypes { + cts := ContentTypes{} + for _, ct := range m { + if !cts.Has(ct) { + cts = append(cts, ct) + } + } + return cts +}*/ diff --git a/content/structerr/structerr.go b/content/structerr/structerr.go new file mode 100644 index 0000000..aa33e21 --- /dev/null +++ b/content/structerr/structerr.go @@ -0,0 +1,20 @@ +package structerr + +// StructError is an interface for errors that are structs, and can be compared for +// errors.Is equality. Equality is determined by type equality, that is, if the pointer +// receiver is *MyError and target can be successfully casted using target.(*MyError), +// then target and the pointer receiver error are equal, otherwise not. +// +// This is needed because errors.Is does not support equality like this for structs +// by default. +type StructError interface { + error + Is(target error) bool +} + +// UnwrapError extends the StructError with a method to retrieve the underlying +// error cause, as per the interface presented in errors.Unwrap. +type UnwrapError interface { + StructError + Unwrap() error +} diff --git a/json/alias.go b/json/alias.go new file mode 100644 index 0000000..b91e052 --- /dev/null +++ b/json/alias.go @@ -0,0 +1,18 @@ +package json + +import ( + "bytes" + "encoding/json" +) + +func Indent(buf *bytes.Buffer, src []byte, prefix, indent string) error { + return json.Indent(buf, src, prefix, indent) +} + +type RawMessage = json.RawMessage + +// Number is a re-export of encoding/json.Number. It can be returned from decoded +// unstructured interface{} targets if DecoderOptions.UnknownNumberStrategy is +// UnknownNumberStrategyJSONNumber. +// TODO: Is this number completely round-trippable? +type Number = json.Number diff --git a/json/decoder.go b/json/decoder.go new file mode 100644 index 0000000..b34bde1 --- /dev/null +++ b/json/decoder.go @@ -0,0 +1,200 @@ +package json + +import ( + "bytes" + "io" + "sync" + + jsoniter "github.com/json-iterator/go" + "github.com/luxas/deklarative/content" +) + +var ( + /*_ content.OptionDisallowUnknownFields = &DecoderOptions{} + _ content.OptionUnknownNumberStrategy = &DecoderOptions{} + + _ content.OptionDisallowUnknownFields = &Decoder{} + _ content.OptionUnknownNumberStrategy = &Decoder{}*/ + _ content.Decoder = &Decoder{} + _ content.FrameDecoder = &Decoder{} +) + +type DecoderOption interface { + applyToDecoder(*DecoderOptions) +} + +func DisallowUnknownFields() DecoderOption { + return &DecoderOptions{UnknownFieldsPolicy: content.UnknownFieldsPolicyError} +} + +func defaultDecoderOpts() *DecoderOptions { + return &DecoderOptions{ + // match encoding/json default + UnknownFieldsPolicy: content.UnknownFieldsPolicyIgnore, + // match yaml.v3 default + DuplicateFieldsPolicy: content.DuplicateFieldsPolicyError, + // match Kubernetes API machinery default + UnknownNumberStrategy: content.UnknownNumberStrategyInt64OrFloat64, + } +} + +type DecoderOptions struct { + // DisallowUnknownFields disallows decoding serialized data into structs where + // the serialized data contain fields not present in the struct. + // If the decode target is an interface{}, any field names are allowed. + // + // Default: UnknownFieldsPolicyIgnore, i.e. unknown fields in the given data are ignored. + // + // TODO: Make an example showing both cases. + UnknownFieldsPolicy content.UnknownFieldsPolicy + + // DisallowDuplicateFields disallows duplicate fields. + // + // Default: DuplicateFieldsPolicyError, i.e. duplicate fields raise an error. + DuplicateFieldsPolicy content.DuplicateFieldsPolicy + + // UnknownNumberStrategy controls how JSON numbers are decoded into interface{} targets. + // + // UnknownNumberStrategyAlwaysFloat64 all JSON numbers where the target type is unknown + // will be decoded into float64's. This can, however, lead to loss of precision, as + // not every integer can be expressed as a float64. For example, try decoding the + // integer 10000000000000001 into a float64, and encoding it again. It will be encoded + // as 10000000000000000; which examplifies the loss of round-trippability. + // + // Hence, there is UnknownNumberStrategyInt64OrFloat64, which first tries to parse a + // JSON number of unknown type as an int64, and only if that fails, assign it to a + // float64. The result is that integers that fit into int64 can always be round-tripped. + // + // UnknownNumberStrategyJSONNumber is the same as encoding/json.Decoder.UseNumber(); + // it leaves the number as a typed string, json.Number; preserving the exact textual + // representation. + // + // Default: UnknownNumberStrategyInt64OrFloat64, which makes integers round-trippable + // by default. + // + // TODO: Make an example showing all three cases. + UnknownNumberStrategy content.UnknownNumberStrategy + + // content.OptionCaseSensitiveGetter is always case-sensitive + // content.OptionDefaultFieldNamingGetter is always fieldName + // content.OptionDisallowDuplicateFieldsGetter is always true +} + +func (o *DecoderOptions) applyToDecoder(target *DecoderOptions) { + if content.ValidUnknownFieldsPolicy(o.UnknownFieldsPolicy) { + target.UnknownFieldsPolicy = o.UnknownFieldsPolicy + } + if content.ValidDuplicateFieldsPolicy(o.DuplicateFieldsPolicy) { + target.DuplicateFieldsPolicy = o.DuplicateFieldsPolicy + } + if content.ValidUnknownNumberStrategy(o.UnknownNumberStrategy) { + target.UnknownNumberStrategy = o.UnknownNumberStrategy + } +} + +func (o *DecoderOptions) applyOptions(opts []DecoderOption) *DecoderOptions { + for _, opt := range opts { + opt.applyToDecoder(o) + } + return o +} + +func (o *DecoderOptions) applyToJSONIterConfig(c *jsoniterConfig) { + c.unknownFieldsPolicy = o.UnknownFieldsPolicy + c.duplicateFieldsPolicy = o.DuplicateFieldsPolicy + c.unknownNumberStrategy = o.UnknownNumberStrategy +} + +func (o *DecoderOptions) toJSONIter() jsoniter.API { + c := jsoniterConfig{} + // apply the default encoding options for the given API + defaultEncoderOpts().applyToJSONIterConfig(&c) + // apply all contained decoding options + o.applyToJSONIterConfig(&c) + return jsoniterForConfig(c) +} + +// NewDecoder creates a new *Decoder +func NewDecoder(r io.Reader, opts ...DecoderOption) *Decoder { + return &Decoder{ + opts: *defaultDecoderOpts().applyOptions(opts), + once: &sync.Once{}, + r: r, + } +} + +// Once the first Decode call is called, the decoder configuration doesn't change. +// +// TODO: Highlight differences between this and encoding/json: +// - Uses json-iter (faster!) +// - Disallows duplicate fields; actually it doesn't!! +// - Can decode maps with int, float64 and bool keys +// - Defaults to decoding int or float for roundtrips +// - Case-sensitive +// +// TODO: Open issues for Kubernetes: +// - The default serializer doesn't apply unknown fields restriction for +// decode into (applies to both YAML and JSON). +// - The default serializer doesn't apply duplicate field restriction when +// JSON-only, because json-iter doesn't support this. +type Decoder struct { + opts DecoderOptions + once *sync.Once + r io.Reader + + d *jsoniter.Decoder +} + +func (d *Decoder) SupportedContentTypes() content.ContentTypes { + return []content.ContentType{content.ContentTypeJSON} +} + +func (d *Decoder) DisallowUnknownFields() { + d.opts.UnknownFieldsPolicy = content.UnknownFieldsPolicyError +} +func (d *Decoder) UseNumber() { + d.opts.UnknownNumberStrategy = content.UnknownNumberStrategyJSONNumber +} + +func (d *Decoder) Decode(obj interface{}) error { + // Initialize d.d once, given the io.Reader and options. + d.once.Do(func() { + d.d = d.opts.toJSONIter().NewDecoder(d.r) + }) + + return d.d.Decode(obj) +} + +func (d *Decoder) DecodeFrame() (content.Frame, error) { + f := &frame{} + if err := d.Decode(&f.content); err != nil { + return nil, err + } + if err := Unmarshal(f.content, &f.obj, &d.opts); err != nil { + return nil, err + } + return f, nil +} + +func Unmarshal(data []byte, v interface{}, opts ...DecoderOption) error { + o := defaultDecoderOpts().applyOptions(opts) + return o.toJSONIter().Unmarshal(data, v) +} + +func UnmarshalStrict(data []byte, v interface{}, opts ...DecoderOption) error { + return Unmarshal(data, v, append(opts, DisallowUnknownFields())...) +} + +var _ content.Frame = &frame{} + +type frame struct { + content RawMessage + obj interface{} +} + +func (f *frame) ContentType() content.ContentType { return content.ContentTypeJSON } +func (f *frame) Content() []byte { return f.content } +func (f *frame) DecodedGeneric() interface{} { return f.obj } +func (f *frame) IsEmpty() bool { return bytes.Equal(f.content, nullBytes) } + +var nullBytes = []byte("null") diff --git a/json/encoder.go b/json/encoder.go new file mode 100644 index 0000000..73184a1 --- /dev/null +++ b/json/encoder.go @@ -0,0 +1,133 @@ +package json + +import ( + "bytes" + "io" + "sync" + + jsoniter "github.com/json-iterator/go" + "github.com/luxas/deklarative/content" +) + +type EncoderOption interface { + applyToEncoder(*EncoderOptions) +} + +func defaultEncoderOpts() *EncoderOptions { + return &EncoderOptions{ + // match encoding/json default + EscapeHTML: boolVar(true), + // match yaml.v3 default + ZeroEncodePolicy: content.CheckIsZero | + content.CheckIsZeroPointer | + content.CheckIsZeroStructRecursive, + } +} +func boolVar(b bool) *bool { return &b } + +type EncoderOptions struct { + Indent string + Prefix string + EscapeHTML *bool + ZeroEncodePolicy content.ZeroEncodePolicy +} + +func (o *EncoderOptions) applyToEncoder(target *EncoderOptions) { + if len(o.Indent) != 0 { + target.Indent = o.Indent + } + if len(o.Prefix) != 0 { + target.Prefix = o.Prefix + } + if o.EscapeHTML != nil { + target.EscapeHTML = o.EscapeHTML + } + if content.IsValidZeroEncodePolicy(o.ZeroEncodePolicy) { + target.ZeroEncodePolicy = o.ZeroEncodePolicy + } +} + +func (o *EncoderOptions) applyOptions(opts []EncoderOption) *EncoderOptions { + for _, opt := range opts { + opt.applyToEncoder(o) + } + return o +} + +func (o *EncoderOptions) applyToJSONIterConfig(c *jsoniterConfig) { + c.escapeHTML = *o.EscapeHTML + c.zeroEncodePolicy = o.ZeroEncodePolicy +} + +func (o *EncoderOptions) toJSONIter() jsoniter.API { + c := jsoniterConfig{} + // apply the default decoding options for the given API + defaultDecoderOpts().applyToJSONIterConfig(&c) + // apply all contained encoding options + o.applyToJSONIterConfig(&c) + return jsoniterForConfig(c) +} + +type Encoder struct { + opts EncoderOptions + once *sync.Once + w io.Writer + + e *jsoniter.Encoder +} + +func (e *Encoder) SetIndent(prefix, indent string) { + e.opts.Prefix = prefix + e.opts.Indent = indent +} + +func (e *Encoder) SetEscapeHTML(escape bool) { + e.opts.EscapeHTML = &escape +} + +func (e *Encoder) Encode(obj interface{}) error { + e.once.Do(func() { + e.e = e.opts.toJSONIter().NewEncoder(e.w) + }) + + // Quick path: encode without any indentation + if len(e.opts.Prefix) == 0 && len(e.opts.Indent) == 0 { + return e.e.Encode(obj) + } + + // Slower path, marshal, then + out, err := MarshalIndent(obj, e.opts.Prefix, e.opts.Indent, &e.opts) + if err != nil { + return err + } + + // TODO: Is this safe to do? Shall a newline be appended? + _, err = e.w.Write(out) + return err + //return e.e.Encode(RawMessage(out)) +} + +func Marshal(obj interface{}, opts ...EncoderOption) ([]byte, error) { + o := defaultEncoderOpts().applyOptions(opts) + return o.toJSONIter().Marshal(obj) +} +func MarshalIndent(obj interface{}, prefix, indent string, opts ...EncoderOption) ([]byte, error) { + // Marshal "the normal way" first; without any indentation + // (even though opts might include indentation settings) + // TODO: If json-iter would marshal e.g. RawMessages correctly, we wouldn't + // need to do this. TODO: Open issues. + out, err := Marshal(obj, opts...) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if _, err := buf.WriteString(prefix); err != nil { + return nil, err + } + if err := Indent(&buf, out, prefix, indent); err != nil { + return nil, err + } + // Return the indented bytes + return buf.Bytes(), nil +} diff --git a/json/example_test.go b/json/example_test.go new file mode 100644 index 0000000..80046bd --- /dev/null +++ b/json/example_test.go @@ -0,0 +1,339 @@ +package json + +import ( + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/luxas/deklarative/content" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func Example_random() { + var obj map[string]interface{} + if err := Unmarshal([]byte(`{"foo": 0.2}`), &obj); err != nil { + fmt.Println(err) + return + } + b, err := Marshal(obj) + if err != nil { + fmt.Println(err) + return + } + fmt.Println(string(b)) + + // Output: + // {"foo":0.2} +} + +func ExampleDecoder_DecodeFrame() { + data := ` { "def" : "bar", "abc": 1 } { "6" : true }["foo", "bar"]"str"123falsetrue1.2{}[]` + + d := NewDecoder(strings.NewReader(data)) + for { + f, err := d.DecodeFrame() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + + fmt.Printf("%s: \"%s\"\n", f.ContentType(), f.Content()) + obj := f.DecodedGeneric() + fmt.Printf("Type: %T, %v\n", obj, obj) + } + + // Output: + // application/json: "{ "def" : "bar", "abc": 1 }" + // Type: map[string]interface {}, map[abc:1 def:bar] + // application/json: "{ "6" : true }" + // Type: map[string]interface {}, map[6:true] + // application/json: "["foo", "bar"]" + // Type: []interface {}, [foo bar] + // application/json: ""str"" + // Type: string, str + // application/json: "123" + // Type: int64, 123 + // application/json: "false" + // Type: bool, false + // application/json: "true" + // Type: bool, true + // application/json: "1.2" + // Type: float64, 1.2 + // application/json: "{}" + // Type: map[string]interface {}, map[] + // application/json: "[]" + // Type: []interface {}, [] +} + +func ExampleDecoder_UseNumber() { + data := ` { "def" : 1, "abc": 1.2 }{"6":1}123"s"3.14{}` + d := NewDecoder(strings.NewReader(data)) + d.UseNumber() + for { + var obj interface{} + err := d.Decode(&obj) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + + numberInfo := func(n Number, indent string) { + i64, err := n.Int64() + fmt.Printf("%sjson.Number.Int64() = %d, %v\n", indent, i64, err) + f64, err := n.Float64() + fmt.Printf("%sjson.Number.Float64() = %f, %v\n", indent, f64, err) + } + + fmt.Printf("Type: %T, %v\n", obj, obj) + switch o := obj.(type) { + case map[string]interface{}: + for k, v := range o { + if n, ok := v.(Number); ok { + fmt.Printf(" Field %s:\n", k) + numberInfo(n, " ") + } + } + case Number: + numberInfo(o, " ") + } + } + + // Output: + // Type: map[string]interface {}, map[abc:1.2 def:1] + // Field def: + // json.Number.Int64() = 1, + // json.Number.Float64() = 1.000000, + // Field abc: + // json.Number.Int64() = 0, strconv.ParseInt: parsing "1.2": invalid syntax + // json.Number.Float64() = 1.200000, + // Type: map[string]interface {}, map[6:1] + // Field 6: + // json.Number.Int64() = 1, + // json.Number.Float64() = 1.000000, + // Type: json.Number, 123 + // json.Number.Int64() = 123, + // json.Number.Float64() = 123.000000, + // Type: string, s + // Type: json.Number, 3.14 + // json.Number.Int64() = 0, strconv.ParseInt: parsing "3.14": invalid syntax + // json.Number.Float64() = 3.140000, + // Type: map[string]interface {}, map[] +} + +func ExampleUnknownNumberStrategyAlwaysFloat64() { + data := ` { "def" : 1, "abc": 1.2 }{"6":1}123"s"3.14{}` + opt := &DecoderOptions{ + UnknownNumberStrategy: content.UnknownNumberStrategyAlwaysFloat64, + } + d1 := NewDecoder(strings.NewReader(data), opt) + for { + var obj interface{} + err := d1.Decode(&obj) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + + fmt.Printf("Type: %T, %v\n", obj, obj) + if o, ok := obj.(map[string]interface{}); ok { + for k, v := range o { + fmt.Printf(" Field %s: %T, %v\n", k, v, v) + } + } + } + + // Output: + // Type: map[string]interface {}, map[abc:1.2 def:1] + // Field def: float64, 1 + // Field abc: float64, 1.2 + // Type: map[string]interface {}, map[6:1] + // Field 6: float64, 1 + // Type: float64, 123 + // Type: string, s + // Type: float64, 3.14 + // Type: map[string]interface {}, map[] +} + +type Target struct { + FieldA string `json:"fieldA"` +} + +func Example_caseSensitiveness() { + t := &Target{} + err := Unmarshal([]byte(`{"fieldA": "value"}`), t) + fmt.Printf("Allowing unknown fields: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = Unmarshal([]byte(`{"fielda": "value"}`), t) + fmt.Printf("Allowing unknown fields: 'fielda' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = UnmarshalStrict([]byte(`{"fielda": "value"}`), t) + fmt.Printf("Disallowing unknown fields: 'fielda' = %q, err = %v\n", t.FieldA, err) + + // Output: + // Allowing unknown fields: 'fieldA' = "value", err = + // Allowing unknown fields: 'fielda' = "", err = + // Disallowing unknown fields: 'fielda' = "", err = json.Target.ReadObject: found unknown field: fielda, error found in #9 byte of ...|{"fielda": "value"}|..., bigger context ...|{"fielda": "value"}|... +} + +func Example_duplicateFields() { + t := &Target{} + err := Unmarshal([]byte(`{"fieldA": "value", "fielda": "val2"}`), t) + fmt.Printf("Case-sensitiveness means no duplicates: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = UnmarshalStrict([]byte(`{"fieldA": "value", "fieldA": "val2"}`), t) + fmt.Printf("Got duplicates in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + t = &Target{} + err = Unmarshal([]byte(`{"fielda": "value", "fielda": "val2"}`), t) + fmt.Printf("Got duplicates not in struct field: 'fieldA' = %q, err = %v\n", t.FieldA, err) + + // Output: + // Allowing unknown fields: 'fieldA' = "value", err = + // Allowing unknown fields: 'fielda' = "", err = + // Disallowing unknown fields: 'fielda' = "", err = json.Target.ReadObject: found unknown field: fielda, error found in #9 byte of ...|{"fielda": "value"}|..., bigger context ...|{"fielda": "value"}|... +} + +type empty struct { + S string `json:"s,omitempty"` +} + +func (e *empty) IsZero() bool { return e.S == "" } +func Example_encode() { + type foo struct { + T time.Time `json:"t,omitempty"` + Str string `json:"str,omitempty"` + Empty empty `json:"empty,omitempty"` + } + out, err := Marshal(foo{Empty: empty{""}}) + fmt.Printf("%s, %v\n", out, err) + + o := metav1.ObjectMeta{CreationTimestamp: metav1.Now()} + out, err = Marshal(o) + fmt.Printf("%s, %v\n", out, err) + + o = metav1.ObjectMeta{} + out, err = Marshal(o) + fmt.Printf("%s, %v\n", out, err) + + // Output: + // foo +} + +func ExampleMarshalIndent() { + type T struct { + String string `json:"string"` + Raw RawMessage `json:"raw"` + } + + t := &T{"Foo", []byte(`{"hello": true}`)} + out, err := MarshalIndent(t, " ", " ") + if err != nil { + fmt.Println(err) + } + fmt.Println("MarshalIndent() =") + fmt.Println(string(out)) + + // Output: + // MarshalIndent() = + // { + // "string": "Foo", + // "raw": { + // "hello": true + // } + // } +} + +/* +func ExampleDecoder_DecodeFrame_float64Only() { + data := ` { "def" : "bar", "abc": 1 } { "6" : true }["foo", "bar"]"str"123falsetrue1.2{}[]` + + d := NewDecoder(strings.NewReader(data), UnknownNumberStrategyAlwaysFloat64()) + for { + f, err := d.DecodeFrame() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + + fmt.Printf("%s: \"%s\"\n", f.ContentType(), f.Content()) + obj := f.DecodedGeneric() + fmt.Printf("Type: %T, %v\n", obj, obj) + } + + // Output: + // application/json: "{ "def" : "bar", "abc": 1 }" + // Type: map[string]interface {}, map[abc:1 def:bar] + // application/json: "{ "6" : true }" + // Type: map[string]interface {}, map[6:true] + // application/json: "["foo", "bar"]" + // Type: []interface {}, [foo bar] + // application/json: ""str"" + // Type: string, str + // application/json: "123" + // Type: float64, 123 + // application/json: "false" + // Type: bool, false + // application/json: "true" + // Type: bool, true + // application/json: "1.2" + // Type: float64, 1.2 + // application/json: "{}" + // Type: map[string]interface {}, map[] + // application/json: "[]" + // Type: []interface {}, [] +} + +func ExampleDecoder_DecodeFrame_jsonNumber() { + data := ` { "def" : "bar", "abc": 1 } { "6" : true }["foo", "bar"]"str"123falsetrue1.2{}[]` + + d := NewDecoder(strings.NewReader(data)) + d.UseNumber() + for { + f, err := d.DecodeFrame() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + panic(err) + } + + fmt.Printf("%s: \"%s\"\n", f.ContentType(), f.Content()) + obj := f.DecodedGeneric() + fmt.Printf("Type: %T, %v\n", obj, obj) + } + + // Output: + // application/json: "{ "def" : "bar", "abc": 1 }" + // Type: map[string]interface {}, map[abc:1 def:bar] + // application/json: "{ "6" : true }" + // Type: map[string]interface {}, map[6:true] + // application/json: "["foo", "bar"]" + // Type: []interface {}, [foo bar] + // application/json: ""str"" + // Type: string, str + // application/json: "123" + // Type: json.Number, 123 + // application/json: "false" + // Type: bool, false + // application/json: "true" + // Type: bool, true + // application/json: "1.2" + // Type: json.Number, 1.2 + // application/json: "{}" + // Type: map[string]interface {}, map[] + // application/json: "[]" + // Type: []interface {}, [] +}*/ diff --git a/json/go.mod b/json/go.mod new file mode 100644 index 0000000..e2efcd0 --- /dev/null +++ b/json/go.mod @@ -0,0 +1,13 @@ +module github.com/luxas/deklarative/json + +go 1.16 + +replace github.com/luxas/deklarative/content => ../content + +require ( + github.com/json-iterator/go v1.1.11 + github.com/luxas/deklarative/content v0.0.0-00010101000000-000000000000 + github.com/modern-go/reflect2 v1.0.1 + github.com/stretchr/testify v1.7.0 + k8s.io/apimachinery v0.22.1 +) diff --git a/json/go.sum b/json/go.sum new file mode 100644 index 0000000..5d01947 --- /dev/null +++ b/json/go.sum @@ -0,0 +1,219 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/json/json.go b/json/json.go new file mode 100644 index 0000000..c362ed6 --- /dev/null +++ b/json/json.go @@ -0,0 +1,3 @@ +package json + +// TODO: Possibly its useful to encode canonical JSON as well? diff --git a/json/json_iter.go b/json/json_iter.go new file mode 100644 index 0000000..de6244d --- /dev/null +++ b/json/json_iter.go @@ -0,0 +1,175 @@ +package json + +import ( + "reflect" + "strconv" + "sync" + "unsafe" + + jsoniter "github.com/json-iterator/go" + "github.com/luxas/deklarative/content" + "github.com/modern-go/reflect2" +) + +// NOTE: This is in part copied from +// https://github.com/kubernetes/apimachinery/blob/v0.22.0/pkg/runtime/serializer/json/json.go#L113-L184 + +type customExtension struct { + jsoniter.DummyExtension + + decodeInt64OrFloat64 bool + zeroEncodePolicy content.ZeroEncodePolicy +} + +func (e *customExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { + // Only return something custom if enabled + if !e.decodeInt64OrFloat64 { + return nil + } + + if typ.String() == "interface {}" { + return customNumberDecoder{} + } + // This really feels like a hack, but allows users to decode into a named + // interface{} alias, e.g. type Generic interface{} in this package. + type1 := typ.Type1() + if typ.Kind() == reflect.Interface && type1 != nil && type1.NumMethod() == 0 { + return customNumberDecoder{} + } + return nil +} + +var isZeroerType = reflect2.TypeOfPtr((*content.IsZeroer)(nil)).Elem() + +func (e *customExtension) DecorateEncoder(typ reflect2.Type, encoder jsoniter.ValEncoder) jsoniter.ValEncoder { + // Opt-in-only + if !content.IsValidZeroEncodePolicy(e.zeroEncodePolicy) { + return encoder + } + + enableIsZeroer := e.zeroEncodePolicy&content.CheckIsZero != 0 + if enableIsZeroer && typ.Implements(isZeroerType) { + return &isZeroValEncoder{encoder, typ} + } + + enableIsZeroerPtr := e.zeroEncodePolicy&content.CheckIsZeroPointer != 0 + ptrType := reflect2.PtrTo(typ) + if enableIsZeroerPtr && ptrType.Implements(isZeroerType) { + return &referenceEncoder{&isZeroValEncoder{encoder, ptrType}} + } + + // TODO: Make this check recursive for structs; somehow. + // Maybe pass the jsoniter.API to the composite ValEncoder implementation + // such that, at runtime, the (cached) ValEncoder for the fields can be + // looked up from the struct's type fields. The logic would be along the lines of: + /* + - If the type here is of kind Struct; return a ValEncoder that + contains the fields of the struct (statically); then at runtime + uses the jsoniter.API to get the right ValEncoder for the right + type (these can even be cached), and then finally uses a + field.UnsafeGet before calling the ValEncoder.IsEmpty function. + */ + return encoder +} + +type referenceEncoder struct{ underlying jsoniter.ValEncoder } + +func (e *referenceEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + e.underlying.Encode(ptr, stream) +} + +func (e *referenceEncoder) IsEmpty(ptr unsafe.Pointer) bool { + return e.underlying.IsEmpty(unsafe.Pointer(&ptr)) +} + +type isZeroValEncoder struct { + underlying jsoniter.ValEncoder + valType reflect2.Type +} + +func (e *isZeroValEncoder) IsEmpty(ptr unsafe.Pointer) bool { + obj := e.valType.UnsafeIndirect(ptr) + if e.valType.IsNullable() && reflect2.IsNil(obj) { + return true + } + return (obj).(content.IsZeroer).IsZero() +} +func (e *isZeroValEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + e.underlying.Encode(ptr, stream) +} + +// TODO: Maybe make this a composite/"decorated" encoder? That might +// be faster/more efficient/correct? +type customNumberDecoder struct{} + +func (customNumberDecoder) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + switch iter.WhatIsNext() { //nolint:exhaustive + case jsoniter.NumberValue: + var number jsoniter.Number + iter.ReadVal(&number) + i64, err := strconv.ParseInt(string(number), 10, 64) + if err == nil { + *(*interface{})(ptr) = i64 + return + } + f64, err := strconv.ParseFloat(string(number), 64) + if err == nil { + *(*interface{})(ptr) = f64 + return + } + iter.ReportError("DecodeNumber", err.Error()) + default: + *(*interface{})(ptr) = iter.Read() + } +} + +func buildJSONIterAPI(c *jsoniterConfig) jsoniter.API { + useNumber := c.unknownNumberStrategy == content.UnknownNumberStrategyJSONNumber + decodeInt64OrFloat := c.unknownNumberStrategy == content.UnknownNumberStrategyInt64OrFloat64 + disallowUnknownFields := c.unknownFieldsPolicy == content.UnknownFieldsPolicyError + + config := jsoniter.Config{ + EscapeHTML: c.escapeHTML, + SortMapKeys: true, + ValidateJsonRawMessage: true, + CaseSensitive: true, + + DisallowUnknownFields: disallowUnknownFields, + UseNumber: useNumber, + }.Froze() + + config.RegisterExtension(&customExtension{ + decodeInt64OrFloat64: decodeInt64OrFloat, + zeroEncodePolicy: c.zeroEncodePolicy, + }) + return config +} + +var ( + jsoniterPool = map[jsoniterConfig]jsoniter.API{} + jsoniterPoolMu = &sync.Mutex{} +) + +// This struct MUST be comparable by value with other structs of the same type, +// and hence, not contain any pointers, maps or slices ("reference types"). +type jsoniterConfig struct { + unknownFieldsPolicy content.UnknownFieldsPolicy + duplicateFieldsPolicy content.DuplicateFieldsPolicy + unknownNumberStrategy content.UnknownNumberStrategy + + escapeHTML bool + zeroEncodePolicy content.ZeroEncodePolicy +} + +func jsoniterForConfig(c jsoniterConfig) jsoniter.API { + jsoniterPoolMu.Lock() + defer jsoniterPoolMu.Unlock() + + // Return cached API if exists for the config + if api, ok := jsoniterPool[c]; ok { + return api + } + + jsoniterPool[c] = buildJSONIterAPI(&c) + return jsoniterPool[c] +} diff --git a/json/json_iter_test.go b/json/json_iter_test.go new file mode 100644 index 0000000..f96426d --- /dev/null +++ b/json/json_iter_test.go @@ -0,0 +1,60 @@ +package json + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" +) + +type myStruct struct { + DEF myCustom + ABC myCustom +} + +type myCustom struct{} + +func (myCustom) MarshalJSON() ([]byte, error) { + return []byte(`{"foo": "bar", "bar": 1}`), nil +} + +func TestMarshal(t *testing.T) { + obj := myStruct{} + out, _ := jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(obj, "", " ") + t.Error(string(out)) + + var buf bytes.Buffer + _ = json.Indent(&buf, out, "", " ") + t.Error(buf.String()) +} + +func TestLossyFloatJSON(t *testing.T) { + obj := map[string]interface{}{} + foo := `{"a":1000000000000000001}` + /*assert.Nil(t, json.Unmarshal([]byte(foo), &obj)) + out, err := json.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out))*/ + + d := NewDecoder(strings.NewReader(foo)) + assert.Nil(t, d.Decode(&obj)) + t.Logf("%T", obj["a"]) + out, err := Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + + obj = map[string]interface{}{} + assert.Nil(t, Unmarshal([]byte(foo), &obj)) + out, err = Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + t.Error() + + /*assert.Nil(t, jsoniter.Unmarshal([]byte(foo), &obj)) + out, err = jsoniter.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out))*/ +} diff --git a/json/options.go b/json/options.go new file mode 100644 index 0000000..a5b981c --- /dev/null +++ b/json/options.go @@ -0,0 +1 @@ +package json diff --git a/json/recognize.go b/json/recognize.go new file mode 100644 index 0000000..aea0565 --- /dev/null +++ b/json/recognize.go @@ -0,0 +1,44 @@ +package json + +import ( + "bytes" + "unicode" + + "github.com/luxas/deklarative/content" +) + +const Recognizer = recognizer(0) + +var _ content.PeekRecognizer = Recognizer + +type recognizer int + +func (recognizer) FromPeekBytes(peek []byte) content.ContentType { + if isJSONBuffer(peek) { + return content.ContentTypeJSON + } + return "" +} + +func (recognizer) SupportedContentTypes() content.ContentTypes { + return []content.ContentType{content.ContentTypeJSON} +} + +// isJSONBuffer scans the provided buffer, looking +// for an open brace indicating this is JSON. +func isJSONBuffer(buf []byte) bool { + return hasJSONPrefix(buf) +} + +// hasJSONPrefix returns true if the provided buffer appears to start with +// a JSON open brace. +func hasJSONPrefix(buf []byte) bool { + return hasPrefix(buf, []byte("{")) +} + +// Return true if the first non-whitespace bytes in buf is +// prefix. +func hasPrefix(buf []byte, prefix []byte) bool { + trim := bytes.TrimLeftFunc(buf, unicode.IsSpace) + return bytes.HasPrefix(trim, prefix) +} diff --git a/yaml/decoder.go b/yaml/decoder.go new file mode 100644 index 0000000..448576c --- /dev/null +++ b/yaml/decoder.go @@ -0,0 +1,159 @@ +package yaml + +import ( + "io" + "sync" + + "github.com/luxas/deklarative/content" + "github.com/luxas/deklarative/json" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type Node = yaml.Node + +//type RNode = yaml.RNode + +//func NewRNode(n *Node) *RNode { return yaml.NewRNode(n) } + +// TODO: This should maybe be a bit more robust, and check for if the DocumentNode's +// children is zero, and not just check if it null. +func IsEmptyDoc(n *Node) bool { return yaml.IsYNodeEmptyDoc(n) } + +var ( +//_ content.OptionDisallowUnknownFields = &DecoderOptions{} +) + +func defaultDecoderOpts() *DecoderOptions { + return &DecoderOptions{ + // match yaml.v3 default + UnknownFieldsPolicy: content.UnknownFieldsPolicyIgnore, + } +} + +type DecoderOption interface { + applyToDecoder(*DecoderOptions) +} + +type DecoderOptions struct { + UnknownFieldsPolicy content.UnknownFieldsPolicy // default: Ignore + + // AutoRecognizeSeqIndent *bool: TODO + + // SupportNonStringKeys *bool needed? + + // content.OptionCaseSensitiveGetter is always case-sensitive + // content.OptionDefaultFieldNamingGetter is always lowercase + // content.OptionDisallowDuplicateFieldsGetter is always true + // content.OptionUnknownNumberStrategyGetter is always int64 then float64 +} + +func (o *DecoderOptions) applyToDecoder(target *DecoderOptions) { + if content.ValidUnknownFieldsPolicy(o.UnknownFieldsPolicy) { + target.UnknownFieldsPolicy = o.UnknownFieldsPolicy + } +} + +func (o *DecoderOptions) applyOptions(opts []DecoderOption) *DecoderOptions { + for _, opt := range opts { + opt.applyToDecoder(o) + } + return o +} + +func (o *DecoderOptions) toJSONOpts() *json.DecoderOptions { + return &json.DecoderOptions{ + UnknownFieldsPolicy: o.UnknownFieldsPolicy, + DuplicateFieldsPolicy: content.DuplicateFieldsPolicyError, + UnknownNumberStrategy: content.UnknownNumberStrategyInt64OrFloat64, + } +} + +func NewDecoder(r io.Reader, opts []DecoderOption) *Decoder { + return &Decoder{ + opts: *defaultDecoderOpts().applyOptions(opts), + once: &sync.Once{}, + r: r, + } +} + +// Once the first Decode call is called, the decoder configuration doesn't change. +type Decoder struct { + opts DecoderOptions + + once *sync.Once + r io.Reader + + d *yaml.Decoder +} + +func (d *Decoder) KnownFields(knownFieldsOnly bool) { + if knownFieldsOnly { + d.opts.UnknownFieldsPolicy = content.UnknownFieldsPolicyError + } + d.opts.UnknownFieldsPolicy = content.UnknownFieldsPolicyIgnore +} + +func (d *Decoder) Decode(into interface{}) error { + d.once.Do(func() { + d.d = yaml.NewDecoder(d.r) + }) + + // If this is a YAML node; fast-path decode it directly. + if isYAMLNode(into) { + return d.d.Decode(into) + } + + // Convert the YAML to an JSON-able object. + jsonObj, err := yamlUnmarshal(func(into interface{}) error { + return d.d.Decode(into) + }) + + j, err := json.Marshal(jsonObj) + if err != nil { + return err + } + + return json.Unmarshal(j, into, d.opts.toJSONOpts()) +} + +func (d *Decoder) DecodeFrame() (content.Frame, error) { + n := &Node{} + if err := d.Decode(n); err != nil { + return nil, err + } + // TODO: Detect the sequence indentation style, and use it when + // marshalling + seqIndent := yaml.CompactSequenceStyle + + content, err := Marshal(n) // TODO: Apply seqIndent + if err != nil { + return nil, err + } + + return &frame{content, n, seqIndent}, nil +} + +type Frame interface { + content.Frame + + SequenceIndentStyle() SequenceIndentStyle + YAMLNode() *Node +} + +var _ content.Frame = &frame{} + +type frame struct { + content []byte + n *Node + seqIndent SequenceIndentStyle +} + +func (f *frame) ContentType() content.ContentType { return content.ContentTypeJSON } +func (f *frame) Content() []byte { return f.content } +func (f *frame) DecodedGeneric() interface{} { return f.n } +func (f *frame) IsEmpty() bool { return IsEmptyDoc(f.n) } + +func (f *frame) SequenceIndentStyle() SequenceIndentStyle { return f.seqIndent } +func (f *frame) YAMLNode() *Node { return f.n } + +type SequenceIndentStyle = yaml.SequenceIndentStyle diff --git a/yaml/encoder.go b/yaml/encoder.go new file mode 100644 index 0000000..bbd81ba --- /dev/null +++ b/yaml/encoder.go @@ -0,0 +1,10 @@ +package yaml + +// the copy comments package should probably be moved here + +type encoderConfig struct { + // setIndent int -- kyaml doesn't allow us to configure this + // seqStyleIndent + // escapeHTML? -- seems like there's no such option + // sortMapKeys? -- one can manipulate the list of RNodes before marshalling +} diff --git a/yaml/features.go b/yaml/features.go new file mode 100644 index 0000000..72f5087 --- /dev/null +++ b/yaml/features.go @@ -0,0 +1 @@ +package yaml diff --git a/yaml/go.mod b/yaml/go.mod new file mode 100644 index 0000000..860398d --- /dev/null +++ b/yaml/go.mod @@ -0,0 +1,23 @@ +module github.com/luxas/deklarative/yaml + +go 1.16 + +replace ( + github.com/luxas/deklarative/content => ../content + github.com/luxas/deklarative/json => ../json + github.com/luxas/deklarative/tracing => ../tracing +) + +require ( + github.com/google/go-cmp v0.5.6 // indirect + github.com/json-iterator/go v1.1.11 + github.com/luxas/deklarative/content v0.0.0-00010101000000-000000000000 + github.com/luxas/deklarative/json v0.0.0-00010101000000-000000000000 + github.com/stretchr/testify v1.7.0 + gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + k8s.io/apimachinery v0.22.1 + sigs.k8s.io/kustomize/kyaml v0.11.1 + sigs.k8s.io/yaml v1.2.0 +) diff --git a/yaml/go.sum b/yaml/go.sum new file mode 100644 index 0000000..6b02fec --- /dev/null +++ b/yaml/go.sum @@ -0,0 +1,331 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e h1:KLHHjkdQFomZy8+06csTWZ0m1343QqxZhR2LJ1OxCYM= +k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= +sigs.k8s.io/kustomize/kyaml v0.11.1 h1:MWihd9syKG7VQnAzr/OpKb94FvH+cw96nEV1u3it7pI= +sigs.k8s.io/kustomize/kyaml v0.11.1/go.mod h1:FTJxEZ86ScK184NpGSAQcfEqee0nul8oLCK30D47m4E= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/yaml/recognize.go b/yaml/recognize.go new file mode 100644 index 0000000..f90b224 --- /dev/null +++ b/yaml/recognize.go @@ -0,0 +1,64 @@ +package yaml + +import ( + "bufio" + "bytes" + + "github.com/luxas/deklarative/content" + "github.com/luxas/deklarative/json" + "gopkg.in/yaml.v3" +) + +const Recognizer = recognizer(0) + +var _ content.PeekRecognizer = Recognizer + +type recognizer int + +func (recognizer) FromPeekBytes(peek []byte) content.ContentType { + if ct := json.Recognizer.FromPeekBytes(peek); len(ct) != 0 { + return ct + } + if isYAML(peek) { + return content.ContentTypeYAML + } + return "" +} + +func (recognizer) SupportedContentTypes() content.ContentTypes { + cts := json.Recognizer.SupportedContentTypes() + cts = append(cts, content.ContentTypeYAML) + return cts +} + +// TODO: Use the approach of reading as many lines as we can, and then putting +// that into yaml.Unmarshal into a *Node. +func isYAML(peek []byte) bool { + line, err := getLine(peek) + if err != nil { + return false + } + + o := map[string]interface{}{} + err = yaml.Unmarshal(line, &o) + return err == nil && len(o) != 0 +} + +// TODO: Use yaml.LineReader instead? +func getLine(peek []byte) ([]byte, error) { + s := bufio.NewScanner(bytes.NewReader(peek)) + // TODO: Support very long lines? (over 65k bytes?) Probably not + for s.Scan() { + t := bytes.TrimSpace(s.Bytes()) + if len(t) == 0 || bytes.Equal(t, []byte("---")) || bytes.HasPrefix(t, []byte{'#'}) { + continue + } + return t, nil + } + // Return a possible scanning error + if err := s.Err(); err != nil { + return nil, err + } + // + return nil, nil +} diff --git a/yaml/recognize_test.go b/yaml/recognize_test.go new file mode 100644 index 0000000..6f70584 --- /dev/null +++ b/yaml/recognize_test.go @@ -0,0 +1,107 @@ +package yaml + +import ( + "bufio" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +//nolint +func Test_isYAML(t *testing.T) { + tests := []struct { + name string + peek string + want bool + }{ + { + name: "field mapping", + peek: "foo: bar\n", + want: true, + }, + { + name: "spaces and other empty documents", + peek: `--- + + +--- +# Ignore me +--- +# Ignore me +foo: bar`, + want: true, + }, + { + name: "bool", + peek: "foo: true", + want: true, + }, + { + name: "int", + peek: "foo: 5", + want: true, + }, + { + name: "float", + peek: "foo: 5.1", + want: true, + }, + { + name: "float", + peek: "foo: null", + want: true, + }, + { + name: "int mapping", + peek: "8080: {}", + want: true, + }, + { + name: "beginning of struct", + peek: "foo:", + want: true, + }, + { + name: "a long key", + peek: strings.Repeat("a", 1000) + ": true", + want: true, + }, + { + name: "scalar null", + peek: `null`, + }, + { + name: "nothing", + }, + { + name: "line overflow", + peek: strings.Repeat("a", bufio.MaxScanTokenSize) + ": true", + }, + { + name: "list element struct", + peek: "- foo: bar", + }, + { + name: "list element string", + peek: "- foo", + }, + { + name: "scalar string", + peek: `foo`, + }, + { + name: "scalar int", + peek: `5`, + }, + { + name: "scalar float", + peek: `5.1`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isYAML([]byte(tt.peek))) + }) + } +} diff --git a/yaml/testdata/TestEncoder.golden b/yaml/testdata/TestEncoder.golden new file mode 100644 index 0000000..ea7d704 --- /dev/null +++ b/yaml/testdata/TestEncoder.golden @@ -0,0 +1,7 @@ +a: foo +b: 123 +c: 1.2 +--- +a: foo +b: 123 +c: 1.2 diff --git a/yaml/upstream.go b/yaml/upstream.go new file mode 100644 index 0000000..81dda7a --- /dev/null +++ b/yaml/upstream.go @@ -0,0 +1,227 @@ +package yaml + +import ( + "fmt" + + "github.com/luxas/deklarative/content" + "github.com/luxas/deklarative/json" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type MarshalOptions struct { + ZeroEncodePolicy content.ZeroEncodePolicy +} + +// Marshal marshals the object into JSON then converts JSON to YAML and returns the +// YAML. +func Marshal(obj interface{}) ([]byte, error) { + // Be compatible with e.g. kyaml.Marshal. + // TODO: What to do here if a Node is embedded within the object? + if isYAMLNode(obj) { + return yamlMarshal(obj) + } + + // Marshal using JSON, to keep struct tags. + j, err := json.Marshal(obj) + if err != nil { + return nil, fmt.Errorf("error marshaling into JSON: %v", err) + } + + y, err := JSONToYAML(j) + if err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + return y, nil +} + +type JSONToYAMLOptions struct { + // DuplicateFieldsPolicy is always Error; it's invalid YAML + + // TODO: Would the UseNumber option even be valid here? + // UnknownNumberStrategy is always Int64OrFloat64; yaml.v3 default + + EncoderOptions +} + +type EncoderOptions struct { + // Do not expose the indent option now; to avoid fragmentation atm. + + kyaml.EncoderOptions + + // TODO: Option for auto-detecting the indent style? + // It should probably be an enum then, with an extra option +} + +// JSONToYAML Converts JSON to YAML. +// If there are duplicate fields in the input JSON, an error will be returned. +func JSONToYAML(j []byte) ([]byte, error) { + // Convert the JSON to an object. + var obj interface{} + // The disallow known fields option here doesn't matter as we're marshalling + // into an interface{}. + // TODO: Always error on duplicate fields + // TODO: Always UnknownNumberStrategyInt64OrFloat64 + if err := json.Unmarshal(j, &obj); err != nil { + return nil, err + } + + // Marshal this object into YAML. + // TODO: Marshal using kyaml's sequence indent option + return yamlMarshal(obj) +} + +func isYAMLNode(obj interface{}) bool { + _, isNode := obj.(Node) + _, isNodePtr := obj.(*Node) + return isNode || isNodePtr +} + +func yamlMarshal(obj interface{}) ([]byte, error) { + return kyaml.MarshalWithOptions(obj, &kyaml.EncoderOptions{ + SeqIndent: kyaml.CompactSequenceStyle, + }) +} + +type UnmarshalOptions struct { + UnknownFieldsPolicy content.UnknownFieldsPolicy +} + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(y []byte, into interface{}) error { + return unmarshal(y, into, false) +} + +// UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal +// into an object, optionally configuring the behavior of the JSON unmarshal. +func UnmarshalStrict(y []byte, into interface{}) error { + return unmarshal(y, into, true) +} + +// unmarshal unmarshals the given YAML byte stream into the given interface, +// optionally performing the unmarshalling strictly +func unmarshal(y []byte, into interface{}, strict bool) error { + // If this is a YAML node; fast-path decode it directly. + if isYAMLNode(into) { + return kyaml.Unmarshal(y, into) + } + + // No need to provide any options here, not for the time being at least, + // as cosmetic changes aren't needed anyways. + j, err := YAMLToJSON(y) + if err != nil { + return fmt.Errorf("error converting YAML to JSON: %v", err) + } + + // TODO: json decoder options here! + // Always disallow duplicate fields, and enable int64orfloat64 + if err := json.Unmarshal(j, into); err != nil { + return fmt.Errorf("error unmarshaling JSON: %v", err) + } + + return nil +} + +type YAMLToJSONOptions struct { + JSONEncoderOpts json.EncoderOptions +} + +// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, +// passing JSON through this method should be a no-op. +// +// Things YAML can do that are not supported by JSON: +// * In YAML you can have binary and null keys in your maps. These are invalid +// in JSON. (int and float keys are converted to strings.) +// * Binary data in YAML with the !!binary tag is not supported. If you want to +// use binary data with this library, encode the data as base64 as usual but do +// not use the !!binary tag in your YAML. This will ensure the original base64 +// encoded data makes it all the way through to the JSON. +// +// If there are duplicate fields, an error is always returned. +// TODO: Rewrite this spec. +func YAMLToJSON(y []byte) ([]byte, error) { + // The disallow unknown fields option doesn't matter here as the decode + // target is an interface{}. + jsonObj, err := yamlUnmarshalBytes(y) + if err != nil { + return nil, err + } + + // Convert this object to JSON and return the data. + // TODO: JSON encode options here + return json.Marshal(jsonObj) +} + +func yamlUnmarshal(decodeFn func(into interface{}) error) (interface{}, error) { + var yamlObj interface{} + if err := decodeFn(yamlObj); err != nil { + return nil, err + } + + return convertNonStringMapKeys(yamlObj) +} + +func yamlUnmarshalBytes(y []byte) (interface{}, error) { + return yamlUnmarshal(func(into interface{}) error { + return kyaml.Unmarshal(y, into) + }) +} + +func ToJSONGeneric(n *Node) (interface{}, error) { + return yamlUnmarshal(func(into interface{}) error { + return n.Decode(into) + }) +} + +// convertNonStringMapKeys traverses an unstructured object +// to find any occurrences of map[interface{}]interface{} that it +// can convert to map[string]interface{}, as required by the JSON +// specification. YAML allows non-string keys, hence this "conversion" +// is required. It automatically disregards NaN and Infinity values, +// as per the JSON spec. +// +// This is still round-trippable, as the json.Decoder allows mapping +// string-encoded map keys into ints, floats and booleans at +// decode-time, hence, we're not losing data. +// +// TODO: Consider forcing obj to be a pointer, such that the signature +// for this function can be convertNonStringMapKeys(obj interface{}) error +// (this would probably?) save some memory and time. +func convertNonStringMapKeys(obj interface{}) (interface{}, error) { + var err error + switch m := obj.(type) { + case []interface{}: + for i, v := range m { + m[i], err = convertNonStringMapKeys(v) + if err != nil { + return nil, err + } + } + return m, nil + case map[string]interface{}: + for k, v := range m { + m[k], err = convertNonStringMapKeys(v) + if err != nil { + return nil, err + } + } + return m, nil + case map[interface{}]interface{}: + // json-iter.*Stream.WriteFloat64 has the following logic: + // if math.IsInf(val, 0) || math.IsNaN(val) { // then error } + // which means that unsupported float values in JSON will be + // caught and reported. + marshalled, err := json.Marshal(m) + if err != nil { + return nil, err + } + var retval map[string]interface{} + if err := json.Unmarshal(marshalled, &retval); err != nil { + return nil, err + } + return retval, nil + default: + return obj, nil + } +} diff --git a/yaml/upstream_test.go b/yaml/upstream_test.go new file mode 100644 index 0000000..4fad726 --- /dev/null +++ b/yaml/upstream_test.go @@ -0,0 +1,1212 @@ +package yaml + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "reflect" + "strconv" + "strings" + "testing" + + jsoniter "github.com/json-iterator/go" + "github.com/stretchr/testify/assert" + yamlv2 "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" + utiljson "k8s.io/apimachinery/pkg/util/json" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" + sigsyaml "sigs.k8s.io/yaml" +) + +func TestYAMLMapKey(t *testing.T) { + var n kyaml.Node + assert.Nil(t, kyaml.Unmarshal([]byte(`{"foo": "bar"}: true`), &n)) + f := n.Content[0] + f2 := f.Content[0] + t.Error(f.Tag, f2.Tag, f2.Content[0].Value, f2.Content[1].Value, f.Content[1].Value) + // TODO: What happens if you pass just a yaml.Node, does it error? + // Does yaml.v3 need a pointer? + b, _ := kyaml.Marshal(&n) + t.Error(string(b)) +} + +type A struct { + B `json:",inline"` + C `json:",inline"` +} + +type B struct { + Foo string +} + +func (b *B) UnmarshalJSON(data []byte) error { + var obj map[string]interface{} + if err := json.Unmarshal(data, &obj); err != nil { + return err + } + if obj == nil { + return nil + } + name, _ := obj["msg"].(string) + b.Foo = "foo-" + name + return nil +} + +type C struct { + Name string +} + +func (c *C) UnmarshalJSON(data []byte) error { + var obj map[string]interface{} + if err := json.Unmarshal(data, &obj); err != nil { + return err + } + if obj == nil { + return nil + } + name, _ := obj["msg"].(string) + c.Name = name + return nil +} + +func TestEmbeddedJSON(t *testing.T) { + var a A + assert.Nil(t, json.Unmarshal([]byte(`{"msg": "hello"}`), &a)) + t.Error(a) +} + +func TestOctals(t *testing.T) { + var obj map[string]interface{} + assert.Nil(t, kyaml.Unmarshal([]byte(`foo: 077`), &obj)) + t.Errorf("%T, %v, %T", obj, obj, obj["foo"]) +} + +func TestMultiFrame(t *testing.T) { + data := `--- +foo: bar +--- +bar: "Null" +--- +{ + "foo": "bar", + "foo": "baz" +} +--- +` + d := kyaml.NewDecoder(strings.NewReader(data)) + for { + var obj interface{} + err := d.Decode(&obj) + if errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + + t.Error(obj) + } +} + +func TestRoundtripLiterals(t *testing.T) { + obj := map[string]interface{}{ + "bar1": "~", + "bar2": nil, + "bar3": "null", + "bar4": "Null", + "baz": "hello", + "foo1": true, + "foo2": "True", + "foo3": math.NaN(), + "foo4": "0777", + } + b, err := kyaml.Marshal(obj) + assert.Nil(t, err) + t.Error(string(b)) +} + +func TestEncodeJSON(t *testing.T) { + scheme := runtime.NewScheme() + s := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, scheme, scheme, k8sjson.SerializerOptions{ + Yaml: true, + }) + + var buf bytes.Buffer + assert.Nil(t, s.Encode(&runtime.Unknown{Raw: []byte("foo: bar\n")}, &buf)) + assert.Nil(t, s.Encode(&runtime.Unknown{Raw: []byte("bar: true\n")}, &buf)) + + t.Error(buf.String()) +} + +func TestEscapeHTML(t *testing.T) { + foo := `foo: ` + var obj interface{} + assert.Nil(t, kyaml.Unmarshal([]byte(foo), &obj)) + t.Error(obj) + + out, err := kyaml.Marshal(obj) + assert.Nil(t, err) + t.Error(string(out)) + + var buf bytes.Buffer + e := json.NewEncoder(&buf) + e.SetEscapeHTML(true) + assert.Nil(t, e.Encode(obj)) + t.Error(buf.String()) + + var out2 interface{} + assert.Nil(t, json.NewDecoder(&buf).Decode(&out2)) + t.Error(out2) +} + +var testData = []byte(`{"foo": "bar", "baz": 1234, "is": true, "arr": [], "obj": {}}`) + +type testStruct struct { + Foo string `json:"foo" yaml:"foo"` + Baz int64 `json:"baz" yaml:"baz"` + Is bool `json:"is" yaml:"is"` + Arr []string `json:"arr" yaml:"arr"` + Obj map[string]string `json:"obj" yaml:"obj"` +} + +func ExampleYAMLMarshal() { + ts := testStruct{Foo: "foo", Is: true, Arr: []string{"1", "2"}} + + out, err := Marshal(ts) + if err != nil { + fmt.Println(err) + } + fmt.Println(string(out)) + // Output: + // fo +} + +func BenchmarkJSONUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = json.Unmarshal(testData, obj) + } +} + +func BenchmarkJSONDecoder(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = json.NewDecoder(bytes.NewReader(testData)).Decode(obj) + } +} + +func BenchmarkJSONIterUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = jsoniter.Unmarshal(testData, obj) + } +} + +func BenchmarkJSONIterDecoder(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = jsoniter.NewDecoder(bytes.NewReader(testData)).Decode(obj) + } +} + +func BenchmarkJSONUtilUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = utiljson.Unmarshal(testData, obj) + } +} + +func BenchmarkYAMLv2Unmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = yamlv2.Unmarshal(testData, obj) + } +} + +func BenchmarkYAMLv2Decoder(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = yamlv2.NewDecoder(bytes.NewReader(testData)).Decode(obj) + } +} + +func BenchmarkK8sYAMLUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = sigsyaml.Unmarshal(testData, obj) + } +} + +func BenchmarkK8sYAMLUnmarshalStrict(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = sigsyaml.UnmarshalStrict(testData, obj) + } +} + +func BenchmarkKyamlUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = kyaml.Unmarshal(testData, obj) + } +} + +func BenchmarkKyamlDecoder(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + obj := &testStruct{} + _ = kyaml.NewDecoder(bytes.NewReader(testData)).Decode(obj) + } +} + +func BenchmarkUnstructuredJSONUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = json.Unmarshal(testData, &obj) + } +} + +func BenchmarkUnstructuredJSONIterUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = json.Unmarshal(testData, &obj) + } +} + +func BenchmarkUnstructuredJSONUtilUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = utiljson.Unmarshal(testData, &obj) + } +} + +func BenchmarkUnstructuredYAMLv2Unmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = yamlv2.Unmarshal(testData, &obj) + } +} + +func BenchmarkUnstructuredK8sYAMLUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = sigsyaml.Unmarshal(testData, &obj) + } +} + +func BenchmarkUnstructuredKyamlUnmarshal(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + var obj interface{} + _ = kyaml.Unmarshal(testData, &obj) + } +} + +type nonStringKeyTest struct { + // TIL that + // "The map's key type must either be any string type, an integer, implement json.Unmarshaler, or implement encoding.TextUnmarshaler." + // i.e. that {"6": "foo"} can be unmarshalTestled/marshalled to map[int(64)]string, but not floats or bools + // + // json-iter supports strings, ints, floats and bools as key, and should hence be pretty + // good to deal with YAML. It's always string-encoded as per the JSON spec, but still. + M map[float64]string +} + +func Example_lossyFloat64() { + var obj map[string]interface{} + tests := map[string]int{ + "-2**53": -1 << 53, + "-2**53 - 1": -1<<53 - 1, + "2**53": 1 << 53, + "2**53 + 1": 1<<53 + 1, + } + + for name, num := range tests { + obj = map[string]interface{}{} + data := `{"a":` + strconv.Itoa(num) + `}` + if err := json.Unmarshal([]byte(data), &obj); err != nil { + fmt.Println(err) + return + } + + out, err := json.Marshal(obj) + if err != nil { + fmt.Println(err) + return + } + roundtripped := string(out) + + fmt.Printf("%s can be roundtripped: %t\n", name, roundtripped == data) + fmt.Printf("Want: %s\n", data) + fmt.Printf("Got: %s\n\n", roundtripped) + } + + /*obj = map[string]interface{}{} + data = `{"a":` + strconv.Itoa(int(math.Pow(2, 53))+1) + `}` + if err := json.Unmarshal([]byte(data), &obj); err != nil { + fmt.Println(err) + return + } + + out, err = json.Marshal(obj) + if err != nil { + fmt.Println(err) + return + } + roundtripped = string(out) + + fmt.Printf("2**53 + 1 can be roundtripped: %t\n", roundtripped == data) + fmt.Printf("Want: %s\n", data) + fmt.Printf("Got: %s\n", roundtripped)*/ + + // Output: + // fo +} + +func TestLossyFloatJSON(t *testing.T) { + var obj interface{} + foo := `{"a":` + strconv.Itoa(-int(math.Pow(2, 53))-1) + `}` + assert.Nil(t, json.Unmarshal([]byte(foo), &obj)) + out, err := json.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + + assert.Nil(t, utiljson.Unmarshal([]byte(foo), &obj)) + out, err = utiljson.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + + assert.Nil(t, jsoniter.Unmarshal([]byte(foo), &obj)) + out, err = jsoniter.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) +} + +func TestLossyFloatYAML(t *testing.T) { + var obj map[string]interface{} + // 10000000000000001 + foo := `a: ` + strconv.Itoa(9007199255000000) + "\n" + + assert.Nil(t, yamlv2.Unmarshal([]byte(foo), &obj)) + out, err := yamlv2.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + t.Logf("%T", obj["a"]) + + assert.Nil(t, kyaml.Unmarshal([]byte(foo), &obj)) + out, err = kyaml.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + t.Logf("%T", obj["a"]) + + assert.Nil(t, sigsyaml.Unmarshal([]byte(foo), &obj)) + out, err = sigsyaml.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, foo, string(out)) + t.Logf("%T", obj["a"]) + t.Error() +} + +type empty struct { + S string `json:"s,omitempty" yaml:"s,omitempty"` +} + +func (e *empty) IsZero() bool { return true } + +//func (empty) MarshalJSON() ([]byte, error) { return []byte("null"), nil } +func Example_encode() { + type foo struct { + T metav1.Time `json:"t,omitempty" yaml:"t,omitempty"` + Str string `json:"str,omitempty" yaml:"str,omitempty"` + Empty *empty `json:"empty,omitempty" yaml:"empty,omitempty"` + } + out, err := kyaml.Marshal(foo{T: metav1.Now(), Str: "bl", Empty: &empty{" "}}) + fmt.Printf("%s, %v\n", out, err) + + /*o := metav1.ObjectMeta{CreationTimestamp: metav1.Now()} + out, err = kyaml.Marshal(o) + fmt.Printf("%s, %v\n", out, err) + + o = metav1.ObjectMeta{} + out, err = kyaml.Marshal(o) + fmt.Printf("%s, %v\n", out, err)*/ + + // Output: + // foo +} + +/* +func TestNonStringKeysToJSON(t *testing.T) { + data := `foo: + 8080: + bla: true + true: bla + 4.1: 4.3 +` + var n Node + assert.Nil(t, kyaml.Unmarshal([]byte(data), &n)) + jsonObj, err := ToJSONGeneric(&n) + assert.Nil(t, err) + + showChildren(t, &n) + t.Errorf("%T %s", jsonObj, jsonObj) + fooNode := jsonObj.(map[string]interface{})["foo"] + t.Errorf("%T %s", fooNode, fooNode) + + out, err := jsoniter.Marshal(jsonObj) + assert.Nil(t, err) + t.Error(string(out)) + + jsonObj, err = ConvertToJSONableObject(jsonObj, nil) + assert.Nil(t, err) + + t.Errorf("%T %s", jsonObj, jsonObj) + fooNode = jsonObj.(map[string]interface{})["foo"] + t.Errorf("%T %s", fooNode, fooNode) + +}*/ + +func showChildren(t *testing.T, n *Node) { + if n.Value != "" { + t.Error(n) + } + for _, ch := range n.Content { + showChildren(t, ch) + } +} + +func TestFoo(t *testing.T) { + var obj interface{} + data := "foo: 04:30" + assert.Nil(t, kyaml.Unmarshal([]byte(data), &obj)) + out, err := kyaml.Marshal(obj) + assert.Nil(t, err) + assert.Equal(t, data, string(out)) +} + +func TestNonStringKeys3(t *testing.T) { + obj := &nonStringKeyTest{M: map[float64]string{5: "foo"}} + out, err := utiljson.Marshal(obj) + assert.Nil(t, err) + e, ok := err.(*json.UnmarshalTypeError) + if ok { + t.Log(e.Type) + } + t.Logf("%s", out) + t.Error() +} + +func TestNonStringKeys2(t *testing.T) { + obj := &nonStringKeyTest{} + err := sigsyaml.Unmarshal([]byte(`{"M":{"5.1": "foo"}}`), obj) + assert.Nil(t, err) + e, ok := err.(*json.UnmarshalTypeError) + if ok { + t.Log(e.Type) + } + t.Logf("%v", obj) + t.Error() +} + +func TestNonStringKeys(t *testing.T) { + foo := ` +m: + 6: foo +` + obj := &nonStringKeyTest{} + assert.Nil(t, kyaml.Unmarshal([]byte(foo), obj)) + t.Logf("yaml.v3: %v", obj) + + rn, err := kyaml.Parse(foo) + assert.Nil(t, err) + jsonMap, err := rn.Map() + assert.Nil(t, err) + t.Logf("yaml.v3: %v", jsonMap) + t.Logf("%T %T %s", jsonMap, jsonMap["m"], (jsonMap["m"].(map[interface{}]interface{}))[6]) + newJson, err := utiljson.Marshal(jsonMap) + assert.Nil(t, err) + t.Logf("%s", newJson) + t.Error() +} + +type hasJSONTag struct { + MyStructField string `json:"myStructField" yaml:"myStructField"` +} + +type noJSONTag struct { + MyStructField string +} + +func TestDuplicateKeys(t *testing.T) { + foo := ` +mystructfield: baz +myStructField: bar +MyStructField: foo +` + f := &hasJSONTag{} + assert.Nil(t, kyaml.Unmarshal([]byte(foo), f)) + t.Logf("yaml.v3: %v", f) + f = &hasJSONTag{} + assert.Nil(t, yamlv2.Unmarshal([]byte(foo), f)) + t.Logf("yaml.v2: %v", f) + f = &hasJSONTag{} + assert.Nil(t, sigsyaml.Unmarshal([]byte(foo), f)) + t.Logf("sigs.yaml: %v", f) + + f2 := &noJSONTag{} + assert.Nil(t, kyaml.Unmarshal([]byte(foo), f2)) + t.Logf("yaml.v3: %v", f2) + f2 = &noJSONTag{} + assert.Nil(t, yamlv2.Unmarshal([]byte(foo), f2)) + t.Logf("yaml.v2: %v", f2) + f2 = &noJSONTag{} + assert.Nil(t, sigsyaml.Unmarshal([]byte(foo), f2)) + t.Logf("sigs.yaml: %v", f2) + + t.Error("foo") +} + +func TestEncoder(t *testing.T) { + /*g := filetest.New(t) + defer g.Assert() + defer g.Update() + + objs := []interface{}{ + MarshalTest{A: "foo", B: 123, C: 1.2}, + MarshalTest{A: "foo", B: 123, C: 1.2}, + } + e := kyaml.NewEncoder(g.Add("TestEncoder").Writer()) + for _, obj := range objs { + assert.Nil(t, e.Encode(obj)) + }*/ + + //str := g.Files["TestEncoder"].Buffer.String() + //fmt.Println("data", str) + str := ` +a: foo +b: 123 +c: 1.2 +--- +a: foo +f: + - bla + + - blabla + +b: 1234 + +c: 1.2 + +` + /* + p := kyaml.Parser{Value: str} + rn, err := p.Filter(nil) + assert.Nil(t, err) + fmt.Println(rn.String()) + + rn, err = p.Filter(nil) + assert.Nil(t, err) + fmt.Println(rn.String()) + */ + + d := kyaml.NewDecoder(strings.NewReader(str)) + n := kyaml.Node{} + assert.Nil(t, d.Decode(&n)) + var buf bytes.Buffer + e := kyaml.NewEncoder(&buf) + assert.Nil(t, e.Encode(&n)) + assert.Nil(t, e.Close()) + fmt.Println(buf.String()) + + n = kyaml.Node{} + assert.Nil(t, d.Decode(&n)) + buf.Reset() + e = kyaml.NewEncoder(&buf) + assert.Nil(t, e.Encode(&n)) + assert.Nil(t, e.Close()) + out, err := json.MarshalIndent(n, "", " ") + fmt.Println(string(out), err) + fmt.Println(buf.String()) + + // + //fmt.Println("foo", n) + + //for range objs { + + /*newObj := MarshalTest{} + assert.Nil(t, d.Decode(&newObj)) + assert.Equal(t, obj, newObj)*/ + //var msg yaml.Unmarshaler + //} + t.Error("out") +} + +type MarshalTest struct { + A string + B int64 + // CHANGE: Fixed this to be float64, instead of float32 + C float64 +} + +func TestMarshal(t *testing.T) { + f64String := strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64) + s := MarshalTest{"a", math.MaxInt32, math.MaxFloat64} + e := fmt.Sprintf("A: a\nB: %d\nC: %s\n", math.MaxInt32, f64String) + + // TODO: Test on a struct that has json tags as well. + y, err := Marshal(s) + assert.Nil(t, err) + assert.Equal(t, e, string(y)) + + s2 := UnmarshalSlice{[]NestedSlice{NestedSlice{"abc", strPtr("def")}, NestedSlice{"123", strPtr("456")}}} + e2 := `a: +- b: abc + c: def +- b: "123" + c: "456" +` + y, err = Marshal(s2) + assert.Nil(t, err) + assert.Equal(t, e2, string(y)) +} + +type UnmarshalString struct { + A string `json:"a"` + True int64 `json:"true"` +} + +type UnmarshalStringMap struct { + A map[string]string `json:"a"` +} + +type UnmarshalNestedString struct { + A NestedString `json:"a"` +} + +type NestedString struct { + A string `json:"a"` +} + +type UnmarshalSlice struct { + A []NestedSlice `json:"a"` +} + +type NestedSlice struct { + B string `json:"b"` + C *string `json:"c"` +} + +func TestUnmarshal(t *testing.T) { + y := []byte(`a: "1"`) // TODO: Verify that numbers need to be quoted in k8s + s1 := UnmarshalString{} + e1 := UnmarshalString{A: "1"} + unmarshalTest(t, y, &s1, &e1) + + y = []byte(`a: "true"`) + s1 = UnmarshalString{} + e1 = UnmarshalString{A: "true"} + unmarshalTest(t, y, &s1, &e1) + + y = []byte(`true: 1`) + s1 = UnmarshalString{} + e1 = UnmarshalString{True: 1} + unmarshalTest(t, y, &s1, &e1) + + y = []byte(` +a: + a: "1"`) + s2 := UnmarshalNestedString{} + e2 := UnmarshalNestedString{NestedString{"1"}} + unmarshalTest(t, y, &s2, &e2) + + y = []byte(` +a: +- b: abc + c: def +- b: "123" + c: "456" +`) + s3 := UnmarshalSlice{} + e3 := UnmarshalSlice{[]NestedSlice{NestedSlice{"abc", strPtr("def")}, NestedSlice{"123", strPtr("456")}}} + unmarshalTest(t, y, &s3, &e3) + + y = []byte(` +a: + b: "1"`) + s4 := UnmarshalStringMap{} + e4 := UnmarshalStringMap{map[string]string{"b": "1"}} + unmarshalTest(t, y, &s4, &e4) + + y = []byte(` +a: + name: TestA +b: + name: TestB +`) + type NamedThing struct { + Name string `json:"name"` + } + s5 := map[string]*NamedThing{} + e5 := map[string]*NamedThing{ + "a": &NamedThing{Name: "TestA"}, + "b": &NamedThing{Name: "TestB"}, + } + unmarshalTest(t, y, &s5, &e5) +} + +func unmarshalTest(t *testing.T, y []byte, s, e interface{}) { + t.Helper() + err := Unmarshal(y, s) + if err != nil { + t.Errorf("error unmarshalTesting YAML: %v", err) + } + + if !reflect.DeepEqual(s, e) { + t.Errorf("unmarshalTest YAML was unsuccessful, expected: %+#v, got: %+#v", + e, s) + } +} + +func TestUnmarshalStrict(t *testing.T) { + y := []byte("a: 1") + s1 := UnmarshalString{} + e1 := UnmarshalString{A: "1"} + unmarshalTestStrict(t, y, &s1, &e1) + + y = []byte("a: true") + s1 = UnmarshalString{} + e1 = UnmarshalString{A: "true"} + unmarshalTestStrict(t, y, &s1, &e1) + + y = []byte("true: 1") + s1 = UnmarshalString{} + e1 = UnmarshalString{True: 1} + unmarshalTestStrict(t, y, &s1, &e1) + + y = []byte("a:\n a: 1") + s2 := UnmarshalNestedString{} + e2 := UnmarshalNestedString{NestedString{"1"}} + unmarshalTestStrict(t, y, &s2, &e2) + + y = []byte("a:\n - b: abc\n c: def\n - b: 123\n c: 456\n") + s3 := UnmarshalSlice{} + e3 := UnmarshalSlice{[]NestedSlice{NestedSlice{"abc", strPtr("def")}, NestedSlice{"123", strPtr("456")}}} + unmarshalTestStrict(t, y, &s3, &e3) + + y = []byte("a:\n b: 1") + s4 := UnmarshalStringMap{} + e4 := UnmarshalStringMap{map[string]string{"b": "1"}} + unmarshalTestStrict(t, y, &s4, &e4) + + y = []byte(` +a: + name: TestA +b: + name: TestB +`) + type NamedThing struct { + Name string `json:"name"` + } + s5 := map[string]*NamedThing{} + e5 := map[string]*NamedThing{ + "a": &NamedThing{Name: "TestA"}, + "b": &NamedThing{Name: "TestB"}, + } + unmarshalTest(t, y, &s5, &e5) + + // When using not-so-strict unmarshalTest, we should + // be picking up the ID-1 as the value in the "id" field + y = []byte(` +a: + name: TestA + id: ID-A + id: ID-1 +`) + type NamedThing2 struct { + Name string `json:"name"` + ID string `json:"id"` + } + s6 := map[string]*NamedThing2{} + e6 := map[string]*NamedThing2{ + "a": {Name: "TestA", ID: "ID-1"}, + } + unmarshalTest(t, y, &s6, &e6) +} + +func TestUnmarshalStrictFails(t *testing.T) { + y := []byte("a: true\na: false") + s1 := UnmarshalString{} + unmarshalTestStrictFail(t, y, &s1) + + y = []byte("a:\n - b: abc\n c: 32\n b: 123") + s2 := UnmarshalSlice{} + unmarshalTestStrictFail(t, y, &s2) + + y = []byte("a:\n b: 1\n c: 3") + s3 := UnmarshalStringMap{} + unmarshalTestStrictFail(t, y, &s3) + + type NamedThing struct { + Name string `json:"name"` + ID string `json:"id"` + } + // When using strict unmarshalTest, we should see + // the unmarshalTest fail if there are multiple keys + y = []byte(` +a: + name: TestA + id: ID-A + id: ID-1 +`) + s4 := NamedThing{} + unmarshalTestStrictFail(t, y, &s4) + + // Strict unmarshalTest should fail for unknown fields + y = []byte(` +name: TestB +id: ID-B +unknown: Some-Value +`) + s5 := NamedThing{} + unmarshalTestStrictFail(t, y, &s5) +} + +func unmarshalTestStrict(t *testing.T, y []byte, s, e interface{}) { + err := UnmarshalStrict(y, s) + if err != nil { + t.Errorf("error unmarshalTesting YAML: %v", err) + } + + if !reflect.DeepEqual(s, e) { + t.Errorf("unmarshalTest YAML was unsuccessful, expected: %+#v, got: %+#v", + e, s) + } +} + +func unmarshalTestStrictFail(t *testing.T, y []byte, s interface{}) { + err := UnmarshalStrict(y, s) + if err == nil { + t.Errorf("error unmarshalTesting YAML: %v", err) + } +} + +type Case struct { + input string + output string + // By default we test that reversing the output == input. But if there is a + // difference in the reversed output, you can optionally specify it here. + reverse *string +} + +type RunType int + +const ( + RunTypeJSONToYAML RunType = iota + RunTypeYAMLToJSON +) + +func TestJSONToYAML(t *testing.T) { + cases := []Case{ + { + `{"t":"a"}`, + "t: a\n", + nil, + }, { + `{"t":null}`, + "t: null\n", + nil, + }, + } + + runCases(t, RunTypeJSONToYAML, cases) +} + +func TestYAMLToJSON(t *testing.T) { + cases := []Case{ + { + "t: a\n", + `{"t":"a"}`, + nil, + }, { + "t: \n", + `{"t":null}`, + strPtr("t: null\n"), + }, { + "t: null\n", + `{"t":null}`, + nil, + }, { + "1: a\n", + `{"1":"a"}`, + strPtr("\"1\": a\n"), + }, { + "1000000000000000000000000000000000000: a\n", + `{"1e+36":"a"}`, + strPtr("\"1e+36\": a\n"), + }, { + "1e+36: a\n", + `{"1e+36":"a"}`, + strPtr("\"1e+36\": a\n"), + }, { + "\"1e+36\": a\n", + `{"1e+36":"a"}`, + nil, + }, { + "\"1.2\": a\n", + `{"1.2":"a"}`, + nil, + }, { + "- t: a\n", + `[{"t":"a"}]`, + nil, + }, { + "- t: a\n" + + "- t:\n" + + " b: 1\n" + + " c: 2\n", + `[{"t":"a"},{"t":{"b":1,"c":2}}]`, + nil, + }, { + `[{t: a}, {t: {b: 1, c: 2}}]`, + `[{"t":"a"},{"t":{"b":1,"c":2}}]`, + strPtr("- t: a\n" + + "- t:\n" + + " b: 1\n" + + " c: 2\n"), + }, { + "- t: \n", + `[{"t":null}]`, + strPtr("- t: null\n"), + }, { + "- t: null\n", + `[{"t":null}]`, + nil, + }, + } + + // Cases that should produce errors. + _ = []Case{ + { + "~: a", + `{"null":"a"}`, + nil, + }, { + "a: !!binary gIGC\n", + "{\"a\":\"\x80\x81\x82\"}", + nil, + }, + } + + runCases(t, RunTypeYAMLToJSON, cases) +} + +func runCases(t *testing.T, runType RunType, cases []Case) { + var f func([]byte) ([]byte, error) + var invF func([]byte) ([]byte, error) + var msg string + var invMsg string + if runType == RunTypeJSONToYAML { + f = JSONToYAML + invF = YAMLToJSON + msg = "JSON to YAML" + invMsg = "YAML back to JSON" + } else { + f = YAMLToJSON + invF = JSONToYAML + msg = "YAML to JSON" + invMsg = "JSON back to YAML" + } + + for _, c := range cases { + // Convert the string. + t.Logf("converting %s\n", c.input) + output, err := f([]byte(c.input)) + if err != nil { + t.Errorf("Failed to convert %s, input: `%s`, err: %v", msg, c.input, err) + } + + // Check it against the expected output. + if string(output) != c.output { + t.Errorf("Failed to convert %s, input: `%s`, expected `%s`, got `%s`", + msg, c.input, c.output, string(output)) + } + + // Set the string that we will compare the reversed output to. + reverse := c.input + // If a special reverse string was specified, use that instead. + if c.reverse != nil { + reverse = *c.reverse + } + + // Reverse the output. + input, err := invF(output) + if err != nil { + t.Errorf("Failed to convert %s, input: `%s`, err: %v", invMsg, string(output), err) + } + + // Check the reverse is equal to the input (or to *c.reverse). + if string(input) != reverse { + t.Errorf("Failed to convert %s, input: `%s`, expected `%s`, got `%s`", + invMsg, string(output), reverse, string(input)) + } + } + +} + +// To be able to easily fill in the *Case.reverse string above. +func strPtr(s string) *string { + return &s +} + +func TestYAMLToJSONStrict(t *testing.T) { + const data = ` +foo: bar +foo: baz +` + if _, err := YAMLToJSON([]byte(data)); err != nil { + t.Error("expected YAMLtoJSON to pass on duplicate field names") + } + // TODO + if _, err := YAMLToJSON([]byte(data)); err == nil { + t.Error("expected YAMLtoJSONStrict to fail on duplicate field names") + } +} + +/* +func TestJSONObjectToYAMLObject(t *testing.T) { + const bigUint64 = ((uint64(1) << 63) + 500) / 1000 * 1000 + intOrInt64 := func(i64 int64) interface{} { + if i := int(i64); i64 == int64(i) { + return i + } + return i64 + } + + tests := []struct { + name string + input map[string]interface{} + expected yaml.MapSlice + }{ + {name: "nil", expected: yaml.MapSlice(nil)}, + {name: "empty", input: map[string]interface{}{}, expected: yaml.MapSlice(nil)}, + { + name: "values", + input: map[string]interface{}{ + "nil slice": []interface{}(nil), + "nil map": map[string]interface{}(nil), + "empty slice": []interface{}{}, + "empty map": map[string]interface{}{}, + "bool": true, + "float64": float64(42.1), + "fractionless": float64(42), + "int": int(42), + "int64": int64(42), + "int64 big": float64(math.Pow(2, 62)), + "negative int64 big": -float64(math.Pow(2, 62)), + "map": map[string]interface{}{"foo": "bar"}, + "slice": []interface{}{"foo", "bar"}, + "string": string("foo"), + "uint64 big": bigUint64, + }, + expected: yaml.MapSlice{ + {Key: "nil slice"}, + {Key: "nil map"}, + {Key: "empty slice", Value: []interface{}{}}, + {Key: "empty map", Value: yaml.MapSlice(nil)}, + {Key: "bool", Value: true}, + {Key: "float64", Value: float64(42.1)}, + {Key: "fractionless", Value: int(42)}, + {Key: "int", Value: int(42)}, + {Key: "int64", Value: int(42)}, + {Key: "int64 big", Value: intOrInt64(int64(1) << 62)}, + {Key: "negative int64 big", Value: intOrInt64(-(1 << 62))}, + {Key: "map", Value: yaml.MapSlice{{Key: "foo", Value: "bar"}}}, + {Key: "slice", Value: []interface{}{"foo", "bar"}}, + {Key: "string", Value: string("foo")}, + {Key: "uint64 big", Value: bigUint64}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := JSONObjectToYAMLObject(tt.input) + sortMapSlicesInPlace(tt.expected) + sortMapSlicesInPlace(got) + if !reflect.DeepEqual(got, tt.expected) { + t.Errorf("jsonToYAML() = %v, want %v", spew.Sdump(got), spew.Sdump(tt.expected)) + } + + jsonBytes, err := json.Marshal(tt.input) + if err != nil { + t.Fatalf("unexpected json.Marshal error: %v", err) + } + var gotByRoundtrip yaml.MapSlice + if err := yaml.Unmarshal(jsonBytes, &gotByRoundtrip); err != nil { + t.Fatalf("unexpected yaml.Unmarshal error: %v", err) + } + + // yaml.Unmarshal loses precision, it's rounding to the 4th last digit. + // Replicate this here in the test, but don't change the type. + for i := range got { + switch got[i].Key { + case "int64 big", "uint64 big", "negative int64 big": + switch v := got[i].Value.(type) { + case int64: + d := int64(500) + if v < 0 { + d = -500 + } + got[i].Value = int64((v+d)/1000) * 1000 + case uint64: + got[i].Value = uint64((v+500)/1000) * 1000 + case int: + d := int(500) + if v < 0 { + d = -500 + } + got[i].Value = int((v+d)/1000) * 1000 + default: + t.Fatalf("unexpected type for key %s: %v:%T", got[i].Key, v, v) + } + } + } + + if !reflect.DeepEqual(got, gotByRoundtrip) { + t.Errorf("yaml.Unmarshal(json.Marshal(tt.input)) = %v, want %v\njson: %s", spew.Sdump(gotByRoundtrip), spew.Sdump(got), string(jsonBytes)) + } + }) + } +} + +func sortMapSlicesInPlace(x interface{}) { + switch x := x.(type) { + case []interface{}: + for i := range x { + sortMapSlicesInPlace(x[i]) + } + case yaml.MapSlice: + sort.Slice(x, func(a, b int) bool { + return x[a].Key.(string) < x[b].Key.(string) + }) + } +} +*/