diff --git a/.changelog/unreleased/dependencies/42-use-grocksdb.md b/.changelog/unreleased/dependencies/42-use-grocksdb.md new file mode 100644 index 0000000..3af697b --- /dev/null +++ b/.changelog/unreleased/dependencies/42-use-grocksdb.md @@ -0,0 +1 @@ +- Switch rocksdb binding from gorocksdb to grocksdb, bump librocksdb dependency to `v7.10.2`. diff --git a/Makefile b/Makefile index 25f52cd..2a3e649 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ test-badgerdb: test-all: @echo "--> Running go test" - @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,badgerdb -v + @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb -v .PHONY: test-all test-all-with-coverage: @@ -54,7 +54,7 @@ test-all-with-coverage: -race \ -coverprofile=coverage.txt \ -covermode=atomic \ - -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,badgerdb \ + -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb \ -v .PHONY: test-all-with-coverage diff --git a/go.mod b/go.mod index e51deb5..fd0016a 100644 --- a/go.mod +++ b/go.mod @@ -7,9 +7,9 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/google/btree v1.1.2 github.com/jmhodges/levigo v1.0.0 + github.com/linxGnu/grocksdb v1.7.16 github.com/stretchr/testify v1.8.2 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c go.etcd.io/bbolt v1.3.7 google.golang.org/grpc v1.53.0 ) diff --git a/go.sum b/go.sum index f04d763..60610ba 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cosmos/gorocksdb v1.1.1 h1:N0OqpEKXgsi2qtDm8T1+AlNMXkTm6s1jowYf7/4pH5I= -github.com/cosmos/gorocksdb v1.1.1/go.mod h1:b/U29r/CtguX3TF7mKG1Jjn4APDqh4wECshxXdiWHpA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -21,12 +19,6 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczC github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= -github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= -github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= -github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -67,6 +59,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/linxGnu/grocksdb v1.7.16 h1:Q2co1xrpdkr5Hx3Fp+f+f7fRGhQFQhvi/+226dtLmA8= +github.com/linxGnu/grocksdb v1.7.16/go.mod h1:JkS7pl5qWpGpuVb3bPqTz8nC12X3YtPZT+Xq7+QfQo4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -99,7 +93,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= diff --git a/rocksdb.go b/rocksdb.go index d0d49e5..945491c 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -8,7 +8,7 @@ import ( "path/filepath" "runtime" - "github.com/tecbot/gorocksdb" + "github.com/linxGnu/grocksdb" ) func init() { @@ -20,10 +20,10 @@ func init() { // RocksDB is a RocksDB backend. type RocksDB struct { - db *gorocksdb.DB - ro *gorocksdb.ReadOptions - wo *gorocksdb.WriteOptions - woSync *gorocksdb.WriteOptions + db *grocksdb.DB + ro *grocksdb.ReadOptions + wo *grocksdb.WriteOptions + woSync *grocksdb.WriteOptions } var _ DB = (*RocksDB)(nil) @@ -32,11 +32,11 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) { // default rocksdb option, good enough for most cases, including heavy workloads. // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). // compression: snappy as default, need to -lsnappy to enable. - bbto := gorocksdb.NewDefaultBlockBasedTableOptions() - bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30)) - bbto.SetFilterPolicy(gorocksdb.NewBloomFilter(10)) + bbto := grocksdb.NewDefaultBlockBasedTableOptions() + bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30)) + bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10)) - opts := gorocksdb.NewDefaultOptions() + opts := grocksdb.NewDefaultOptions() opts.SetBlockBasedTableFactory(bbto) opts.SetCreateIfMissing(true) opts.IncreaseParallelism(runtime.NumCPU()) @@ -45,23 +45,26 @@ func NewRocksDB(name string, dir string) (*RocksDB, error) { return NewRocksDBWithOptions(name, dir, opts) } -func NewRocksDBWithOptions(name string, dir string, opts *gorocksdb.Options) (*RocksDB, error) { +func NewRocksDBWithOptions(name string, dir string, opts *grocksdb.Options) (*RocksDB, error) { dbPath := filepath.Join(dir, name+".db") - db, err := gorocksdb.OpenDb(opts, dbPath) + db, err := grocksdb.OpenDb(opts, dbPath) if err != nil { return nil, err } - ro := gorocksdb.NewDefaultReadOptions() - wo := gorocksdb.NewDefaultWriteOptions() - woSync := gorocksdb.NewDefaultWriteOptions() + ro := grocksdb.NewDefaultReadOptions() + wo := grocksdb.NewDefaultWriteOptions() + woSync := grocksdb.NewDefaultWriteOptions() woSync.SetSync(true) - database := &RocksDB{ + return NewRocksDBWithRawDB(db, ro, wo, woSync), nil +} + +func NewRocksDBWithRawDB(db *grocksdb.DB, ro *grocksdb.ReadOptions, wo *grocksdb.WriteOptions, woSync *grocksdb.WriteOptions) *RocksDB { + return &RocksDB{ db: db, ro: ro, wo: wo, woSync: woSync, } - return database, nil } // Get implements DB. @@ -139,7 +142,7 @@ func (db *RocksDB) DeleteSync(key []byte) error { return nil } -func (db *RocksDB) DB() *gorocksdb.DB { +func (db *RocksDB) DB() *grocksdb.DB { return db.db } diff --git a/rocksdb_batch.go b/rocksdb_batch.go index ff5abbf..6ebc8da 100644 --- a/rocksdb_batch.go +++ b/rocksdb_batch.go @@ -3,11 +3,11 @@ package db -import "github.com/tecbot/gorocksdb" +import "github.com/linxGnu/grocksdb" type rocksDBBatch struct { db *RocksDB - batch *gorocksdb.WriteBatch + batch *grocksdb.WriteBatch } var _ Batch = (*rocksDBBatch)(nil) @@ -15,7 +15,7 @@ var _ Batch = (*rocksDBBatch)(nil) func newRocksDBBatch(db *RocksDB) *rocksDBBatch { return &rocksDBBatch{ db: db, - batch: gorocksdb.NewWriteBatch(), + batch: grocksdb.NewWriteBatch(), } } diff --git a/rocksdb_iterator.go b/rocksdb_iterator.go index fb19ebe..495517c 100644 --- a/rocksdb_iterator.go +++ b/rocksdb_iterator.go @@ -6,11 +6,11 @@ package db import ( "bytes" - "github.com/tecbot/gorocksdb" + "github.com/linxGnu/grocksdb" ) type rocksDBIterator struct { - source *gorocksdb.Iterator + source *grocksdb.Iterator start, end []byte isReverse bool isInvalid bool @@ -18,7 +18,7 @@ type rocksDBIterator struct { var _ Iterator = (*rocksDBIterator)(nil) -func newRocksDBIterator(source *gorocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { +func newRocksDBIterator(source *grocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { if isReverse { if end == nil { source.SeekToLast() @@ -136,7 +136,7 @@ func (itr *rocksDBIterator) assertIsValid() { // moveSliceToBytes will free the slice and copy out a go []byte // This function can be applied on *Slice returned from Key() and Value() // of an Iterator, because they are marked as freed. -func moveSliceToBytes(s *gorocksdb.Slice) []byte { +func moveSliceToBytes(s *grocksdb.Slice) []byte { defer s.Free() if !s.Exists() { return nil diff --git a/tools/Dockerfile b/tools/Dockerfile index 56330fa..1d50e48 100644 --- a/tools/Dockerfile +++ b/tools/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ FROM build AS install ARG LEVELDB=1.20 -ARG ROCKSDB=6.24.2 +ARG ROCKSDB=7.10.2 # Install cleveldb RUN \