diff --git a/go.mod b/go.mod index 50da60d45..184c88687 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,8 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 - github.com/golang/snappy v0.0.3 github.com/google/flatbuffers v1.12.1 - github.com/klauspost/compress v1.12.3 + github.com/klauspost/compress v1.15.15 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 5939d6fcc..7c755a8b9 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -37,8 +35,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/table/builder.go b/table/builder.go index 5c9e065e0..bf0ac319a 100644 --- a/table/builder.go +++ b/table/builder.go @@ -25,8 +25,8 @@ import ( "unsafe" "github.com/golang/protobuf/proto" - "github.com/golang/snappy" fbs "github.com/google/flatbuffers/go" + "github.com/klauspost/compress/s2" "github.com/pkg/errors" "github.com/dgraph-io/badger/v4/fb" @@ -159,7 +159,7 @@ func NewTableBuilder(opts Options) *Builder { func maxEncodedLen(ctype options.CompressionType, sz int) int { switch ctype { case options.Snappy: - return snappy.MaxEncodedLen(sz) + return s2.MaxEncodedLen(sz) case options.ZSTD: return y.ZSTDCompressBound(sz) } @@ -523,9 +523,9 @@ func (b *Builder) compressData(data []byte) ([]byte, error) { case options.None: return data, nil case options.Snappy: - sz := snappy.MaxEncodedLen(len(data)) + sz := s2.MaxEncodedLen(len(data)) dst := b.alloc.Allocate(sz) - return snappy.Encode(dst, data), nil + return s2.EncodeSnappy(dst, data), nil case options.ZSTD: sz := y.ZSTDCompressBound(len(data)) dst := b.alloc.Allocate(sz) diff --git a/table/builder_test.go b/table/builder_test.go index 0045b8887..86907a6ab 100644 --- a/table/builder_test.go +++ b/table/builder_test.go @@ -178,12 +178,15 @@ func BenchmarkBuilder(b *testing.B) { opt.BlockSize = 4 * 1024 opt.BloomFalsePositive = 0.01 opt.TableSize = 5 << 20 + b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { builder := NewTableBuilder(*opt) for j := 0; j < keysCount; j++ { builder.Add(keyList[j], vs, 0) } + _ = builder.Finish() builder.Close() } @@ -208,6 +211,11 @@ func BenchmarkBuilder(b *testing.B) { opt.DataKey = &pb.DataKey{Data: key} bench(b, &opt) }) + b.Run("snappy compression", func(b *testing.B) { + var opt Options + opt.Compression = options.Snappy + bench(b, &opt) + }) b.Run("zstd compression", func(b *testing.B) { var opt Options opt.Compression = options.ZSTD diff --git a/table/table.go b/table/table.go index 0bbc91089..010cbd1cf 100644 --- a/table/table.go +++ b/table/table.go @@ -32,7 +32,8 @@ import ( "unsafe" "github.com/golang/protobuf/proto" - "github.com/golang/snappy" + "github.com/klauspost/compress/snappy" + "github.com/klauspost/compress/zstd" "github.com/pkg/errors" "github.com/dgraph-io/badger/v4/fb" @@ -818,6 +819,11 @@ func (t *Table) decompress(b *block) error { } case options.ZSTD: sz := int(float64(t.opt.BlockSize) * 1.2) + // Get frame content size from header. + var hdr zstd.Header + if err := hdr.Decode(b.data); err == nil && hdr.HasFCS && hdr.FrameContentSize < uint64(t.opt.BlockSize*2) { + sz = int(hdr.FrameContentSize) + } dst = z.Calloc(sz, "Table.Decompress") b.data, err = y.ZSTDDecompress(dst, b.data) if err != nil { diff --git a/value_test.go b/value_test.go index 25af06eda..cbdcf4793 100644 --- a/value_test.go +++ b/value_test.go @@ -977,7 +977,7 @@ func BenchmarkReadWrite(b *testing.B) { opts.ValueThreshold = 0 db, err := Open(opts) y.Check(err) - + defer db.Close() vl := &db.vlog b.ResetTimer()