Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use a buffer pool in report.ReadBinary() to reduce garbage-collection #3255

Merged
merged 2 commits into from
Jul 4, 2018
+17 −8
Merged

Use a buffer pool in report.ReadBinary() to reduce garbage-collection #3255

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
84169c6
Use a buffer pool in report.ReadBinary() to reduce garbage-collection
bboreham Jul 4, 2018
8e5b19c
Use buffer pool in report.MakeFromBytes() to reduce garbage-collection
bboreham Jul 4, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions report/marshal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"sync"

log "github.com/Sirupsen/logrus"
"github.com/ugorji/go/codec"
Expand Down Expand Up @@ -61,13 +61,18 @@ func (c byteCounter) Read(p []byte) (n int, err error) {
return n, err
}

// buffer pool to reduce garbage-collection
var bufferPool = &sync.Pool{
New: func() interface{} { return new(bytes.Buffer) },
}

// ReadBinary reads bytes into a Report.
//
// Will decompress the binary if gzipped is true, and will use the given
// codecHandle to decode it.
func (rep *Report) ReadBinary(r io.Reader, gzipped bool, codecHandle codec.Handle) error {
var err error
var compressedSize, uncompressedSize uint64
var compressedSize uint64

// We have historically had trouble with reports being too large. We are
// keeping this instrumentation around to help us implement
Expand All @@ -82,12 +87,14 @@ func (rep *Report) ReadBinary(r io.Reader, gzipped bool, codecHandle codec.Handl
}
}
// Read everything into memory before decoding: it's faster
buf, err := ioutil.ReadAll(r)
buf := bufferPool.Get().(*bytes.Buffer)
buf.Reset()
defer bufferPool.Put(buf)
uncompressedSize, err := buf.ReadFrom(r)
if err != nil {
return err
}
uncompressedSize = uint64(len(buf))
if err := rep.ReadBytes(buf, codecHandle); err != nil {
if err := rep.ReadBytes(buf.Bytes(), codecHandle); err != nil {
return err
}
log.Debugf(
Expand Down Expand Up @@ -120,19 +127,21 @@ func MakeFromBytes(buf []byte) (*Report, error) {
if err != nil {
return nil, err
}
buf, err = ioutil.ReadAll(r)
buffer := bufferPool.Get().(*bytes.Buffer)
buffer.Reset()
defer bufferPool.Put(buffer)
uncompressedSize, err := buffer.ReadFrom(r)
if err != nil {
return nil, err
}
uncompressedSize := len(buf)
log.Debugf(
"Received report sizes: compressed %d bytes, uncompressed %d bytes (%.2f%%)",
compressedSize,
uncompressedSize,
float32(compressedSize)/float32(uncompressedSize)*100,
)
rep := MakeReport()
if err := rep.ReadBytes(buf, &codec.MsgpackHandle{}); err != nil {
if err := rep.ReadBytes(buffer.Bytes(), &codec.MsgpackHandle{}); err != nil {
return nil, err
}
return &rep, nil
Expand Down