Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add Encoder and Decoder #59

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
build:
strategy:
matrix:
go-versions: [1.13.x, 1.14.x, 1.15.x]
go-versions: [1.13.x, 1.14.x, 1.15.x, 1.16.x]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
Expand Down
48 changes: 48 additions & 0 deletions bench_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package yaml

import (
"testing"
)

type benchStruct struct {
a int
b string
c map[string]float32
}

func BenchmarkMarshal(b *testing.B) {
s := benchStruct{
a: 5,
b: "foobar",
c: map[string]float32{
"bish": 1.2,
"bash": 3.4,
"bosh": 5.6,
},
}

b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := Marshal(s); err != nil {
b.Fatal(err)
}
}
}

func BenchmarkUnmarshal(b *testing.B) {
yaml := []byte(`a: 5
b: "foobar"
c:
bish: 1.2
bash: 3.4
bosh: 5.6
`)
var s benchStruct

b.ResetTimer()
for i := 0; i < b.N; i++ {
if err := Unmarshal(yaml, &s); err != nil {
b.Fatal(err)
}
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@ go 1.12

require (
github.com/davecgh/go-spew v1.1.1
gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v2 v2.4.0
)
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
152 changes: 111 additions & 41 deletions yaml.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,98 @@ import (
"gopkg.in/yaml.v2"
)

// Marshal marshals the object into JSON then converts JSON to YAML and returns the
// YAML.
func Marshal(o interface{}) ([]byte, error) {
j, err := json.Marshal(o)
if err != nil {
return nil, fmt.Errorf("error marshaling into JSON: %v", err)
// An Encoder serializes values into YAML and writes them using its io.Writer.
type Encoder struct {
w io.Writer
}

// NewEncoder creates a new Encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w: w}
}

// Encode serializes a value into YAML and writes it using the Encoder's
// io.Writer.
func (e *Encoder) Encode(o interface{}) error {
pipeReader, pipeWriter := io.Pipe()
defer pipeReader.Close()

var intermediate interface{}
jsonEnc := json.NewEncoder(pipeWriter)
// We are using yaml.Decoder here (instead of json.Decoder) because the Go
// JSON library doesn't try to pick the right number type (int, float,
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

json.NewDecoder(...).UseNumber() will preserve numeric input literally (https://pkg.go.dev/encoding/json#Decoder.UseNumber) ... would that enable translating with fidelity? how does the json decoder compare to the yaml decoder in efficiency?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also, a unit test demonstrating some of the edge cases like this would be helpful in ensuring this isn't producing user-visible changes

other examples of weird edges in yaml encoding/decoding we don't want to accidentally modify behavior on (even if the current behavior is not ultimately desired/correct):

// etc.) when unmarshalling to interface{}; it always picks float64.
// go-yaml preserves the number type.
yamlDec := yaml.NewDecoder(pipeReader)
yamlEnc := yaml.NewEncoder(e.w)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is doing an encode/decode/encode? Why? There has to be a better way than this? There should be some comment explaining the purpose?


var jsonErr error
go func() {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

spawning goroutines here and in Decode is a little hard to reason about... a few things I noticed/wondered

  • if this panics, the calling program will exit instead of propagating the panic to the Encode() caller... I'd expect the panic to propagate instead
  • is it better to start the json encoder async and the yaml decoder inline or vice-versa?
  • is it better to leave the pipereader unbuffered or wrap in a buffer?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW, this is adapted from the official io.Pipe docs.

jsonErr = jsonEnc.Encode(o)
pipeWriter.Close()
}()
yamlErr := yamlDec.Decode(&intermediate)
if jsonErr != nil {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm surprised this doesn't complain about races the same way

return fmt.Errorf("error marshalling into JSON: %v", jsonErr)
}
if yamlErr != nil {
return fmt.Errorf("error converting JSON to YAML: %v", yamlErr)
}
if err := yamlEnc.Encode(intermediate); err != nil {
return err
}
return yamlEnc.Close()
}

y, err := JSONToYAML(j)
if err != nil {
return nil, fmt.Errorf("error converting JSON to YAML: %v", err)
// A Decoder reads YAML from its io.Reader and de-serializes it into values.
type Decoder struct {
dec *yaml.Decoder
opts []JSONOpt
}

// NewDecoder creates a new Decoder.
func NewDecoder(r io.Reader, opts ...JSONOpt) *Decoder {
return &Decoder{
dec: yaml.NewDecoder(r),
opts: opts,
}
}

// SetStrict sets whether strict decoding behaviour is enabled when decoding
// items in the data (see UnmarshalStrict). By default, decoding is not strict.
func (d *Decoder) SetStrict(strict bool) {
d.dec.SetStrict(strict)
}

// Decode reads YAML from its io.Reader and de-serializes it into the given
// value.
func (d *Decoder) Decode(o interface{}) error {
vo := reflect.ValueOf(o)
r, w := io.Pipe()
defer r.Close()
// go test -race complains if we set the error via closure (e.g. see Encode).
errChan := make(chan error, 1)
go func() {
errChan <- yamlToJSON(&vo, d.dec, w)
w.Close()
}()
jsonErr := jsonUnmarshal(r, o, d.opts...)
yamlErr := <-errChan
if yamlErr != nil {
return fmt.Errorf("error converting YAML to JSON: %v", yamlErr)
}
if jsonErr != nil {
return fmt.Errorf("error unmarshalling JSON: %v", jsonErr)
}
return nil
}

return y, nil
// Marshal serializes the value provided into a YAML document.
func Marshal(o interface{}) ([]byte, error) {
buf := &bytes.Buffer{}
enc := NewEncoder(buf)
err := enc.Encode(o)
return buf.Bytes(), err
}

// JSONOpt is a decoding option for decoding from JSON format.
Expand All @@ -33,34 +111,18 @@ type JSONOpt func(*json.Decoder) *json.Decoder
// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object,
// optionally configuring the behavior of the JSON unmarshal.
func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
return yamlUnmarshal(y, o, false, opts...)
buf := bytes.NewBuffer(y)
dec := NewDecoder(buf, opts...)
return dec.Decode(o)
}

// UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal
// into an object, optionally configuring the behavior of the JSON unmarshal.
func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error {
return yamlUnmarshal(y, o, true, append(opts, DisallowUnknownFields)...)
}

// yamlUnmarshal unmarshals the given YAML byte stream into the given interface,
// optionally performing the unmarshalling strictly
func yamlUnmarshal(y []byte, o interface{}, strict bool, opts ...JSONOpt) error {
vo := reflect.ValueOf(o)
unmarshalFn := yaml.Unmarshal
if strict {
unmarshalFn = yaml.UnmarshalStrict
}
j, err := yamlToJSON(y, &vo, unmarshalFn)
if err != nil {
return fmt.Errorf("error converting YAML to JSON: %v", err)
}

err = jsonUnmarshal(bytes.NewReader(j), o, opts...)
if err != nil {
return fmt.Errorf("error unmarshaling JSON: %v", err)
}

return nil
buf := bytes.NewBuffer(y)
dec := NewDecoder(buf, append(opts, DisallowUnknownFields)...)
dec.SetStrict(true)
return dec.Decode(o)
}

// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
Expand Down Expand Up @@ -109,21 +171,29 @@ func JSONToYAML(j []byte) ([]byte, error) {
//
// For strict decoding of YAML, use YAMLToJSONStrict.
func YAMLToJSON(y []byte) ([]byte, error) {
return yamlToJSON(y, nil, yaml.Unmarshal)
bufIn := bytes.NewBuffer(y)
bufOut := &bytes.Buffer{}
dec := yaml.NewDecoder(bufIn)
err := yamlToJSON(nil, dec, bufOut)
return bytes.TrimSpace(bufOut.Bytes()), err
}

// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding,
// returning an error on any duplicate field names.
func YAMLToJSONStrict(y []byte) ([]byte, error) {
return yamlToJSON(y, nil, yaml.UnmarshalStrict)
bufIn := bytes.NewBuffer(y)
bufOut := &bytes.Buffer{}
dec := yaml.NewDecoder(bufIn)
dec.SetStrict(true)
err := yamlToJSON(nil, dec, bufOut)
return bytes.TrimSpace(bufOut.Bytes()), err
}

func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) {
func yamlToJSON(jsonTarget *reflect.Value, dec *yaml.Decoder, tgt io.Writer) error {
// Convert the YAML to an object.
var yamlObj interface{}
err := yamlUnmarshal(y, &yamlObj)
if err != nil {
return nil, err
if err := dec.Decode(&yamlObj); err != nil {
return err
}

// YAML objects are not completely compatible with JSON objects (e.g. you
Expand All @@ -132,11 +202,11 @@ func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte,
// incompatibilties happen along the way.
jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget)
if err != nil {
return nil, err
return err
}

// Convert this object to JSON and return the data.
return json.Marshal(jsonObj)
return json.NewEncoder(tgt).Encode(jsonObj)
}

func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) {
Expand Down
2 changes: 1 addition & 1 deletion yaml_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ func unmarshalStrict(t *testing.T, y []byte, s, e interface{}, opts ...JSONOpt)
func unmarshalStrictFail(t *testing.T, y []byte, s interface{}, opts ...JSONOpt) {
err := UnmarshalStrict(y, s, opts...)
if err == nil {
t.Errorf("error unmarshaling YAML: %v", err)
t.Errorf("expected strict unmarshalling to fail:\n%s", y)
}
}

Expand Down