-
Notifications
You must be signed in to change notification settings - Fork 240
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Problem: there's no compact historical state storage
Closes: #704 Solution: - Integration version store and streaming service.
- Loading branch information
Showing
12 changed files
with
610 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| package versiondb | ||
|
|
||
| import ( | ||
| "encoding/binary" | ||
| "sort" | ||
|
|
||
| "github.com/RoaringBitmap/roaring/roaring64" | ||
| ) | ||
|
|
||
| var ChunkLimit = uint64(1950) // threshold beyond which MDBX overflow pages appear: 4096 / 2 - (keySize + 8) | ||
|
|
||
| // CutLeft - cut from bitmap `targetSize` bytes from left | ||
| // removing lft part from `bm` | ||
| // returns nil on zero cardinality | ||
| func CutLeft64(bm *roaring64.Bitmap, sizeLimit uint64) *roaring64.Bitmap { | ||
| if bm.GetCardinality() == 0 { | ||
| return nil | ||
| } | ||
|
|
||
| sz := bm.GetSerializedSizeInBytes() | ||
| if sz <= sizeLimit { | ||
| lft := roaring64.New() | ||
| lft.AddRange(bm.Minimum(), bm.Maximum()+1) | ||
| lft.And(bm) | ||
| lft.RunOptimize() | ||
| bm.Clear() | ||
| return lft | ||
| } | ||
|
|
||
| from := bm.Minimum() | ||
| minMax := bm.Maximum() - bm.Minimum() | ||
| to := sort.Search(int(minMax), func(i int) bool { // can be optimized to avoid "too small steps", but let's leave it for readability | ||
| lft := roaring64.New() // bitmap.Clear() method intentionally not used here, because then serialized size of bitmap getting bigger | ||
| lft.AddRange(from, from+uint64(i)+1) | ||
| lft.And(bm) | ||
| lft.RunOptimize() | ||
| return lft.GetSerializedSizeInBytes() > sizeLimit | ||
| }) | ||
|
|
||
| lft := roaring64.New() | ||
| lft.AddRange(from, from+uint64(to)) // no +1 because sort.Search returns element which is just higher threshold - but we need lower | ||
| lft.And(bm) | ||
| bm.RemoveRange(from, from+uint64(to)) | ||
| lft.RunOptimize() | ||
| return lft | ||
| } | ||
|
|
||
| func WalkChunks64(bm *roaring64.Bitmap, sizeLimit uint64, f func(chunk *roaring64.Bitmap, isLast bool) error) error { | ||
| for bm.GetCardinality() > 0 { | ||
| if err := f(CutLeft64(bm, sizeLimit), bm.GetCardinality() == 0); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func WalkChunkWithKeys64(k []byte, m *roaring64.Bitmap, sizeLimit uint64, f func(chunkKey []byte, chunk *roaring64.Bitmap) error) error { | ||
| return WalkChunks64(m, sizeLimit, func(chunk *roaring64.Bitmap, isLast bool) error { | ||
| chunkKey := make([]byte, len(k)+8) | ||
| copy(chunkKey, k) | ||
| if isLast { | ||
| binary.BigEndian.PutUint64(chunkKey[len(k):], ^uint64(0)) | ||
| } else { | ||
| binary.BigEndian.PutUint64(chunkKey[len(k):], chunk.Maximum()) | ||
| } | ||
| return f(chunkKey, chunk) | ||
| }) | ||
| } | ||
|
|
||
| // SeekInBitmap64 - returns value in bitmap which is >= n | ||
| func SeekInBitmap64(m *roaring64.Bitmap, n uint64) (found uint64, ok bool) { | ||
| if m == nil || m.IsEmpty() { | ||
| return 0, false | ||
| } | ||
| if n == 0 { | ||
| return m.Minimum(), true | ||
| } | ||
| searchRank := m.Rank(n - 1) | ||
| if searchRank >= m.GetCardinality() { | ||
| return 0, false | ||
| } | ||
| found, _ = m.Select(searchRank) | ||
| return found, true | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| package versiondb | ||
|
|
||
| import ( | ||
| "bytes" | ||
|
|
||
| "github.com/RoaringBitmap/roaring/roaring64" | ||
|
|
||
| sdk "github.com/cosmos/cosmos-sdk/types" | ||
| dbm "github.com/tendermint/tm-db" | ||
| ) | ||
|
|
||
| const LastChunkId = ^uint64(0) | ||
|
|
||
| func HistoryIndexKey(key []byte, height uint64) []byte { | ||
| return append(key, sdk.Uint64ToBigEndian(height)...) | ||
| } | ||
|
|
||
| // GetHistoryIndex returns the history index bitmap chunk which covers the target version. | ||
| func GetHistoryIndex(db dbm.DB, key []byte, height uint64) (*roaring64.Bitmap, error) { | ||
| // try to seek the first chunk whose maximum is bigger or equal to the target height. | ||
| it, err := db.Iterator( | ||
| HistoryIndexKey(key, height), | ||
| sdk.PrefixEndBytes(key), | ||
| ) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer it.Close() // nolint: errcheck | ||
|
|
||
| if !it.Valid() { | ||
| return nil, nil | ||
| } | ||
|
|
||
| m := roaring64.New() | ||
| _, err = m.ReadFrom(bytes.NewReader(it.Value())) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| return m, nil | ||
| } | ||
|
|
||
| // SeekHistoryIndex locate the minimal version that changed the key and is larger than the target version, | ||
| // using the returned version can find the value for the target version in changeset table. | ||
| // If not found, return -1 | ||
| func SeekHistoryIndex(db dbm.DB, key []byte, version uint64) (int64, error) { | ||
| // either m.Maximum() >= version + 1, or is the last chunk. | ||
| m, err := GetHistoryIndex(db, key, version+1) | ||
| if err != nil { | ||
| return -1, err | ||
| } | ||
| found, ok := SeekInBitmap64(m, version+1) | ||
| if !ok { | ||
| return -1, nil | ||
| } | ||
| return int64(found), nil | ||
| } | ||
|
|
||
| // WriteHistoryIndex set the block height to the history bitmap. | ||
| // it try to set to the last chunk, if the last chunk exceeds chunk limit, split it. | ||
| func WriteHistoryIndex(db dbm.DB, batch dbm.Batch, key []byte, height uint64) error { | ||
| lastKey := HistoryIndexKey(key, LastChunkId) | ||
| bz, err := db.Get(lastKey) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| m := roaring64.New() | ||
| if len(bz) > 0 { | ||
| _, err = m.ReadFrom(bytes.NewReader(bz)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| m.Add(height) | ||
|
|
||
| // chunking | ||
| if err = WalkChunks64(m, ChunkLimit, func(chunk *roaring64.Bitmap, isLast bool) error { | ||
| chunkKey := lastKey | ||
| if !isLast { | ||
| chunkKey = HistoryIndexKey(key, chunk.Maximum()) | ||
| } | ||
| bz, err := chunk.ToBytes() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return batch.Set(chunkKey, bz) | ||
| }); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
Oops, something went wrong.