Upgrade compress package

Upgrade for faster/better compression: klauspost/compress@v1.12.3...v1.15.15 Use 's2' in Snappy compatible mode for faster/better Snappy (de)compression: https://github.com/klauspost/compress/tree/master/s2#blocks Should improve compression speed, and `MaxEncodedLen` is considerably less. From `n = 32 + n + n/6` to `n + ((log2(n)+7)/7)` (max 10 bytes) This change is both forward and backwards compatible, meaning output will be decompressable by older versions as well. Benchmarks. Most are very noisy and dominated by allocs. ``` benchmark old ns/op new ns/op delta BenchmarkDbGrowth-32 2387593900 2366353800 -0.89% BenchmarkIteratePrefixSingleKey/Key_lookups-32 89209 87230 -2.22% BenchmarkReadWrite/0.1,0064-32 1827 1821 -0.33% BenchmarkReadWrite/0.2,0064-32 1830 1819 -0.60% BenchmarkReadWrite/0.5,0064-32 1873 1820 -2.83% BenchmarkReadWrite/1.0,0064-32 1881 1879 -0.11% BenchmarkReadWrite/0.1,0128-32 1892 1891 -0.05% BenchmarkReadWrite/0.2,0128-32 1920 1895 -1.30% BenchmarkReadWrite/0.5,0128-32 1942 1917 -1.29% BenchmarkReadWrite/1.0,0128-32 2001 1997 -0.20% BenchmarkReadWrite/0.1,0256-32 2039 2067 +1.37% BenchmarkReadWrite/0.2,0256-32 2065 2063 -0.10% BenchmarkReadWrite/0.5,0256-32 2106 2120 +0.66% BenchmarkReadWrite/1.0,0256-32 2211 2270 +2.67% BenchmarkReadWrite/0.1,0512-32 2292 2320 +1.22% BenchmarkReadWrite/0.2,0512-32 2329 2368 +1.67% BenchmarkReadWrite/0.5,0512-32 2450 2449 -0.04% BenchmarkReadWrite/1.0,0512-32 2589 2625 +1.39% BenchmarkReadWrite/0.1,1024-32 2837 2886 +1.73% BenchmarkReadWrite/0.2,1024-32 2960 3001 +1.39% BenchmarkReadWrite/0.5,1024-32 3162 3186 +0.76% BenchmarkReadWrite/1.0,1024-32 3513 3572 +1.68% BenchmarkReadWrite/0.1,2048-32 3857 3911 +1.40% BenchmarkReadWrite/0.2,2048-32 4036 4077 +1.02% BenchmarkReadWrite/0.5,2048-32 4484 4561 +1.72% BenchmarkReadWrite/1.0,2048-32 5193 5203 +0.19% BenchmarkReadWrite/0.1,4096-32 5814 5862 +0.83% BenchmarkReadWrite/0.2,4096-32 6139 6109 -0.49% BenchmarkReadWrite/0.5,4096-32 7136 6972 -2.30% BenchmarkReadWrite/1.0,4096-32 8391 8434 +0.51% BenchmarkReadWrite/0.1,8192-32 9412 9479 +0.71% BenchmarkReadWrite/0.2,8192-32 10088 10118 +0.30% BenchmarkReadWrite/0.5,8192-32 11580 11715 +1.17% BenchmarkReadWrite/1.0,8192-32 14446 14516 +0.48% BenchmarkReadWrite/0.1,16384-32 16798 16754 -0.26% BenchmarkReadWrite/0.2,16384-32 17645 17687 +0.24% BenchmarkReadWrite/0.5,16384-32 21139 21396 +1.22% BenchmarkReadWrite/1.0,16384-32 26156 27147 +3.79% BenchmarkReadWrite/frac_0-32 194 190 -1.81% BenchmarkReadWrite/frac_1-32 168 169 +0.24% BenchmarkReadWrite/frac_2-32 171 160 -6.71% BenchmarkReadWrite/frac_3-32 183 152 -17.18% BenchmarkReadWrite/frac_4-32 131 129 -1.75% BenchmarkReadWrite/frac_5-32 122 122 +0.08% BenchmarkReadWrite/frac_6-32 117 114 -2.40% BenchmarkReadWrite/frac_7-32 97.7 93.0 -4.76% BenchmarkReadWrite/frac_8-32 79.2 68.0 -14.15% BenchmarkReadWrite/frac_9-32 54.6 59.3 +8.57% BenchmarkReadWrite/frac_10-32 6.57 8.01 +21.89% BenchmarkReadWriteMap/frac_0-32 363 369 +1.65% BenchmarkReadWriteMap/frac_1-32 344 343 -0.32% BenchmarkReadWriteMap/frac_2-32 334 338 +1.32% BenchmarkReadWriteMap/frac_3-32 345 341 -1.19% BenchmarkReadWriteMap/frac_4-32 346 350 +1.36% BenchmarkReadWriteMap/frac_5-32 354 360 +1.78% BenchmarkReadWriteMap/frac_6-32 361 357 -1.11% BenchmarkReadWriteMap/frac_7-32 376 374 -0.43% BenchmarkReadWriteMap/frac_8-32 366 364 -0.46% BenchmarkReadWriteMap/frac_9-32 312 310 -0.80% BenchmarkReadWriteMap/frac_10-32 40.1 40.0 -0.32% BenchmarkWrite-32 209 199 -4.83% BenchmarkSortedWrites/builder-32 58.4 62.1 +6.41% BenchmarkSortedWrites/skiplist-32 354 348 -1.84% BenchmarkSortedWrites/buffer-32 81.5 81.5 +0.09% BenchmarkBuilder/no_compression-32 150415843 145178786 -3.48% BenchmarkBuilder/encryption-32 202647000 201539767 -0.55% BenchmarkBuilder/snappy_compression/level_1-32 177067050 171329850 -3.24% BenchmarkBuilder/zstd_compression/level_1-32 178846817 172781467 -3.39% BenchmarkBuilder/zstd_compression/level_3-32 178101100 171115667 -3.92% BenchmarkBuilder/zstd_compression/level_15-32 179374867 174848183 -2.52% BenchmarkRead-32 658641400 483333167 -26.62% BenchmarkReadAndBuild-32 1624537000 1457130300 -10.30% BenchmarkReadMerged-32 1102582000 894604200 -18.86% BenchmarkChecksum/CRC_1024-32 68.6 68.5 -0.12% BenchmarkChecksum/xxHash64_1024-32 69.5 69.8 +0.42% BenchmarkChecksum/SHA256_1024-32 2175 2181 +0.28% BenchmarkChecksum/CRC_2048-32 128 128 +0.00% BenchmarkChecksum/xxHash64_2048-32 131 132 +0.69% BenchmarkChecksum/SHA256_2048-32 4159 4164 +0.12% BenchmarkChecksum/CRC_4096-32 250 251 +0.48% BenchmarkChecksum/xxHash64_4096-32 265 262 -1.39% BenchmarkChecksum/SHA256_4096-32 8135 8112 -0.28% BenchmarkChecksum/CRC_8192-32 499 498 -0.28% BenchmarkChecksum/xxHash64_8192-32 517 514 -0.56% BenchmarkChecksum/SHA256_8192-32 16050 16076 +0.16% BenchmarkChecksum/CRC_16384-32 986 984 -0.21% BenchmarkChecksum/xxHash64_16384-32 1016 1013 -0.30% BenchmarkChecksum/SHA256_16384-32 31963 31920 -0.13% BenchmarkChecksum/CRC_32768-32 1949 1950 +0.05% BenchmarkChecksum/xxHash64_32768-32 2016 2024 +0.40% BenchmarkChecksum/SHA256_32768-32 63541 63775 +0.37% BenchmarkChecksum/CRC_65536-32 3901 3902 +0.03% BenchmarkChecksum/xxHash64_65536-32 4028 4015 -0.32% BenchmarkChecksum/SHA256_65536-32 126903 129392 +1.96% BenchmarkChecksum/CRC_131072-32 7760 7976 +2.78% BenchmarkChecksum/xxHash64_131072-32 8030 8035 +0.06% BenchmarkChecksum/SHA256_131072-32 254342 259402 +1.99% BenchmarkChecksum/CRC_262144-32 15666 15760 +0.60% BenchmarkChecksum/xxHash64_262144-32 16080 16047 -0.21% BenchmarkChecksum/SHA256_262144-32 507193 515186 +1.58% BenchmarkChecksum/CRC_1048576-32 61978 63675 +2.74% BenchmarkChecksum/xxHash64_1048576-32 64210 64711 +0.78% BenchmarkChecksum/SHA256_1048576-32 2029004 2085432 +2.78% BenchmarkRandomRead-32 25890 18761 -27.54% BenchmarkBuffer/bytes-buffer-32 590 592 +0.32% BenchmarkBuffer/page-buffer/page-size-1024-32 164 162 -1.40% benchmark old MB/s new MB/s speedup BenchmarkBuilder/no_compression-32 553.13 573.09 1.04x BenchmarkBuilder/encryption-32 410.57 412.82 1.01x BenchmarkBuilder/snappy_compression/level_1-32 469.88 485.61 1.03x BenchmarkBuilder/zstd_compression/level_1-32 465.20 481.53 1.04x BenchmarkBuilder/zstd_compression/level_3-32 467.15 486.22 1.04x BenchmarkBuilder/zstd_compression/level_15-32 463.83 475.84 1.03x ``` Bugfix: using s2 instead of snappy
hypermodeinc · Sep 27, 2023 · 12d48aa · 12d48aa
1 parent 5f004c4
commit 12d48aa
Show file tree

Hide file tree

Showing 6 changed files with 23 additions and 12 deletions.
diff --git a/go.mod b/go.mod
@@ -8,9 +8,8 @@ require (
 	github.com/dustin/go-humanize v1.0.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang/protobuf v1.5.2
-	github.com/golang/snappy v0.0.3
 	github.com/google/flatbuffers v1.12.1
-	github.com/klauspost/compress v1.12.3
+	github.com/klauspost/compress v1.15.15
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.4.0

diff --git a/go.sum b/go.sum
@@ -26,8 +26,6 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw=
 github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -37,8 +35,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU=
-github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw=
+github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

diff --git a/table/builder.go b/table/builder.go
@@ -25,8 +25,8 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/golang/snappy"
 	fbs "github.com/google/flatbuffers/go"
+	"github.com/klauspost/compress/s2"
 	"github.com/pkg/errors"
 
 	"github.com/dgraph-io/badger/v4/fb"
@@ -159,7 +159,7 @@ func NewTableBuilder(opts Options) *Builder {
 func maxEncodedLen(ctype options.CompressionType, sz int) int {
 	switch ctype {
 	case options.Snappy:
-		return snappy.MaxEncodedLen(sz)
+		return s2.MaxEncodedLen(sz)
 	case options.ZSTD:
 		return y.ZSTDCompressBound(sz)
 	}
@@ -523,9 +523,9 @@ func (b *Builder) compressData(data []byte) ([]byte, error) {
 	case options.None:
 		return data, nil
 	case options.Snappy:
-		sz := snappy.MaxEncodedLen(len(data))
+		sz := s2.MaxEncodedLen(len(data))
 		dst := b.alloc.Allocate(sz)
-		return snappy.Encode(dst, data), nil
+		return s2.EncodeSnappy(dst, data), nil
 	case options.ZSTD:
 		sz := y.ZSTDCompressBound(len(data))
 		dst := b.alloc.Allocate(sz)

diff --git a/table/builder_test.go b/table/builder_test.go
@@ -178,12 +178,15 @@ func BenchmarkBuilder(b *testing.B) {
 		opt.BlockSize = 4 * 1024
 		opt.BloomFalsePositive = 0.01
 		opt.TableSize = 5 << 20
+
 		b.ResetTimer()
+		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
 			builder := NewTableBuilder(*opt)
 			for j := 0; j < keysCount; j++ {
 				builder.Add(keyList[j], vs, 0)
 			}
+
 			_ = builder.Finish()
 			builder.Close()
 		}
@@ -208,6 +211,11 @@ func BenchmarkBuilder(b *testing.B) {
 		opt.DataKey = &pb.DataKey{Data: key}
 		bench(b, &opt)
 	})
+	b.Run("snappy compression", func(b *testing.B) {
+		var opt Options
+		opt.Compression = options.Snappy
+		bench(b, &opt)
+	})
 	b.Run("zstd compression", func(b *testing.B) {
 		var opt Options
 		opt.Compression = options.ZSTD

diff --git a/table/table.go b/table/table.go
@@ -32,7 +32,8 @@ import (
 	"unsafe"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/golang/snappy"
+	"github.com/klauspost/compress/snappy"
+	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
 
 	"github.com/dgraph-io/badger/v4/fb"
@@ -818,6 +819,11 @@ func (t *Table) decompress(b *block) error {
 		}
 	case options.ZSTD:
 		sz := int(float64(t.opt.BlockSize) * 1.2)
+		// Get frame content size from header.
+		var hdr zstd.Header
+		if err := hdr.Decode(b.data); err == nil && hdr.HasFCS && hdr.FrameContentSize < uint64(t.opt.BlockSize*2) {
+			sz = int(hdr.FrameContentSize)
+		}
 		dst = z.Calloc(sz, "Table.Decompress")
 		b.data, err = y.ZSTDDecompress(dst, b.data)
 		if err != nil {

diff --git a/value_test.go b/value_test.go
@@ -977,7 +977,7 @@ func BenchmarkReadWrite(b *testing.B) {
 				opts.ValueThreshold = 0
 				db, err := Open(opts)
 				y.Check(err)
-
+				defer db.Close()
 				vl := &db.vlog
 				b.ResetTimer()