From b0fd355035ddf729eb8b2ddb7ac6d06db399ef42 Mon Sep 17 00:00:00 2001 From: bo Date: Wed, 6 Jan 2021 08:39:18 -0700 Subject: [PATCH 1/3] keep allocations as small as possible --- zstd/seqdec.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/zstd/seqdec.go b/zstd/seqdec.go index b5c8ef1332..2c9ac7940e 100644 --- a/zstd/seqdec.go +++ b/zstd/seqdec.go @@ -181,11 +181,13 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { return fmt.Errorf("output (%d) bigger than max block size", size) } if size > cap(s.out) { - // Not enough size, will be extremely rarely triggered, + // Not enough size, which can happen under high volume block streaming conditions // but could be if destination slice is too small for sync operations. - // We add maxBlockSize to the capacity. - s.out = append(s.out, make([]byte, maxBlockSize)...) - s.out = s.out[:len(s.out)-maxBlockSize] + // over-allocating here can create a large amount of GC pressure so we try to keep + // it as contained as possible + addBytes := size - len(s.out) + s.out = append(s.out, make([]byte, addBytes)...) + s.out = s.out[:len(s.out)-addBytes] } if ml > maxMatchLen { return fmt.Errorf("match len (%d) bigger than max allowed length", ml) From 9fb59a0095a949fee470e78b1beadba4ff24f896 Mon Sep 17 00:00:00 2001 From: bo Date: Wed, 6 Jan 2021 10:37:36 -0700 Subject: [PATCH 2/3] add a little more byte padding --- zstd/seqdec.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zstd/seqdec.go b/zstd/seqdec.go index 2c9ac7940e..7466554fab 100644 --- a/zstd/seqdec.go +++ b/zstd/seqdec.go @@ -185,7 +185,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { // but could be if destination slice is too small for sync operations. // over-allocating here can create a large amount of GC pressure so we try to keep // it as contained as possible - addBytes := size - len(s.out) + addBytes := 1024 + (size - len(s.out)) s.out = append(s.out, make([]byte, addBytes)...) s.out = s.out[:len(s.out)-addBytes] } From 6dbb4dd0a5636225129142d51a61047bfd581e2f Mon Sep 17 00:00:00 2001 From: bo Date: Wed, 6 Jan 2021 11:23:40 -0700 Subject: [PATCH 3/3] suggested change --- zstd/seqdec.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zstd/seqdec.go b/zstd/seqdec.go index 7466554fab..1dd39e63b7 100644 --- a/zstd/seqdec.go +++ b/zstd/seqdec.go @@ -185,7 +185,12 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { // but could be if destination slice is too small for sync operations. // over-allocating here can create a large amount of GC pressure so we try to keep // it as contained as possible - addBytes := 1024 + (size - len(s.out)) + used := len(s.out) - startSize + addBytes := 256 + ll + ml + used>>2 + // Clamp to max block size. + if used+addBytes > maxBlockSize { + addBytes = maxBlockSize - used + } s.out = append(s.out, make([]byte, addBytes)...) s.out = s.out[:len(s.out)-addBytes] }