Skip to content

Commit

Permalink
mdbx: v13 (#13205)
Browse files Browse the repository at this point in the history
  • Loading branch information
@AskAlexSharov
AskAlexSharov authored Dec 24, 2024
1 parent 83faf86 commit 1347676
Show file tree
Hide file tree
Showing 14 changed files with 72 additions and 105 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ GOPRIVATE = github.com/erigontech/silkworm-go

PACKAGE = github.com/erigontech/erigon

GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false
GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false
GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}"

GOBUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build $(GO_FLAGS)
Expand Down
2 changes: 1 addition & 1 deletion core/rawdb/accessors_indexes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ package rawdb

import (
"encoding/binary"

libcommon "github.com/erigontech/erigon-lib/common"
"github.com/erigontech/erigon-lib/kv"
"github.com/erigontech/erigon-lib/log/v3"

"github.com/erigontech/erigon/core/types"
)

Expand Down
9 changes: 4 additions & 5 deletions core/state/intra_block_state_logger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,16 @@ import (
"reflect"
"testing"

"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"

libcommon "github.com/erigontech/erigon-lib/common"
"github.com/erigontech/erigon-lib/kv/rawdbv3"
"github.com/erigontech/erigon-lib/log/v3"
stateLib "github.com/erigontech/erigon-lib/state"

"github.com/erigontech/erigon/core/tracing"
"github.com/erigontech/erigon/core/tracing/mocks"

"github.com/holiman/uint256"
"github.com/stretchr/testify/require"
gomock "go.uber.org/mock/gomock"
)

func TestStateLogger(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions erigon-lib/config3/config3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package config3

// Default number of transactions (txNums) in one "step". One static file can have [1, 64] steps.
const DefaultStepSize = 1_562_500 // = 100M / 64. Dividers: 2, 5, 10, 20, 50, 100, 500
//const DefaultStepSize = 1_562_500 / 10

// StepsInFrozenFile - files of this size are completely frozen/immutable.
// files of smaller size are also immutable, but can be removed after merge to bigger files.
Expand Down
4 changes: 2 additions & 2 deletions erigon-lib/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ replace (
require (
github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e
github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed
github.com/erigontech/mdbx-go v0.38.4
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55
github.com/erigontech/secp256k1 v1.1.0
github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
)
Expand Down Expand Up @@ -62,7 +62,7 @@ require (
require (
github.com/cespare/xxhash v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/runtime-spec v1.2.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions erigon-lib/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp
github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M=
github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed h1:un44S8Tuol4LBIC6R94t93GShM53BYjz7GsNPziDLQ8=
github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE=
github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo=
github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI=
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4=
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk=
github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY=
github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls=
github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U=
Expand Down Expand Up @@ -271,8 +271,8 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq
github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU=
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
Expand Down
2 changes: 1 addition & 1 deletion erigon-lib/kv/mdbx/kv_mdbx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) {

}

env, err := mdbx.NewEnv()
env, err := mdbx.NewEnv(mdbx.Label(opts.label))
if err != nil {
return nil, err
}
Expand Down
58 changes: 0 additions & 58 deletions erigon-lib/kv/mdbx/kv_mdbx_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,64 +458,6 @@ func TestAppendFirstLast(t *testing.T) {
require.Equal(t, []string{"value6.1"}, values)
}

func TestNextPrevCurrent(t *testing.T) {
_, _, c := BaseCase(t)

k, v, err := c.First()
require.Nil(t, err)
keys, values := iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values)

k, v, err = c.Next()
require.Equal(t, []byte("key1"), k)
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values)

k, v, err = c.Current()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values)
require.Equal(t, k, []byte("key1"))
require.Equal(t, v, []byte("value1.3"))

k, v, err = c.Next()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key3", "key3"}, keys)
require.Equal(t, []string{"value3.1", "value3.3"}, values)

k, v, err = c.Prev()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values)

k, v, err = c.Current()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values)

k, v, err = c.Prev()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values)

err = c.DeleteCurrent()
require.Nil(t, err)
k, v, err = c.Current()
require.Nil(t, err)
keys, values = iteration(t, c, k, v)
require.Equal(t, []string{"key1", "key3", "key3"}, keys)
require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values)

}

func TestSeek(t *testing.T) {
_, _, c := BaseCase(t)

Expand Down
44 changes: 31 additions & 13 deletions erigon-lib/state/history_stream.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,10 +283,11 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error {
}
seek = next
}
for k, _, err := hi.valsCDup.Seek(seek); k != nil; k, _, err = hi.valsCDup.NextNoDup() {
if err != nil {
return err
}
k, _, err := hi.valsCDup.Seek(seek)
if err != nil {
return err
}
for k != nil {
if hi.toPrefix != nil && bytes.Compare(k, hi.toPrefix) >= 0 {
break
}
Expand All @@ -295,8 +296,16 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error {
return err
}
if v == nil {
seek, ok := kv.NextSubtree(k)
if !ok {
break
}
if k, _, err = hi.valsCDup.Seek(seek); err != nil {
panic(err)
}
continue
}

hi.nextKey = k
hi.nextVal = v[8:]
return nil
Expand Down Expand Up @@ -566,24 +575,33 @@ func (hi *HistoryChangesIterDB) advanceSmallVals() (err error) {
return err
}
}
for ; k != nil; k, _, err = hi.valsCDup.NextNoDup() {
if err != nil {
return err
}
for k != nil {
v, err := hi.valsCDup.SeekBothRange(k, hi.startTxKey[:])
if err != nil {
return err
}
if v == nil {
next, ok := kv.NextSubtree(k)
if !ok {
hi.nextKey = nil
return nil
}
k, _, err = hi.valsCDup.Seek(next)
if err != nil {
return err
}
continue
}
foundTxNumVal := v[:8]
if hi.endTxNum >= 0 && int(binary.BigEndian.Uint64(foundTxNumVal)) >= hi.endTxNum {
continue
if hi.endTxNum < 0 || int(binary.BigEndian.Uint64(foundTxNumVal)) < hi.endTxNum {
hi.nextKey = k
hi.nextVal = v[8:]
return nil
}
k, _, err = hi.valsCDup.NextNoDup()
if err != nil {
return err
}
hi.nextKey = k
hi.nextVal = v[8:]
return nil
}
hi.nextKey = nil
return nil
Expand Down
14 changes: 6 additions & 8 deletions erigon-lib/state/history_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,25 +28,23 @@ import (
"testing"
"time"

"github.com/erigontech/erigon-lib/common/length"
"github.com/erigontech/erigon-lib/config3"
"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon-lib/seg"

"github.com/erigontech/erigon-lib/common"
"github.com/erigontech/erigon-lib/common/datadir"

"github.com/stretchr/testify/require"
btree2 "github.com/tidwall/btree"

"github.com/erigontech/erigon-lib/common"
"github.com/erigontech/erigon-lib/common/background"
"github.com/erigontech/erigon-lib/common/datadir"
"github.com/erigontech/erigon-lib/common/hexutility"
"github.com/erigontech/erigon-lib/common/length"
"github.com/erigontech/erigon-lib/config3"
"github.com/erigontech/erigon-lib/kv"
"github.com/erigontech/erigon-lib/kv/mdbx"
"github.com/erigontech/erigon-lib/kv/order"
"github.com/erigontech/erigon-lib/kv/stream"
"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon-lib/recsplit"
"github.com/erigontech/erigon-lib/recsplit/eliasfano32"
"github.com/erigontech/erigon-lib/seg"
)

func testDbAndHistory(tb testing.TB, largeValues bool, logger log.Logger) (kv.RwDB, *History) {
Expand Down
18 changes: 13 additions & 5 deletions erigon-lib/state/inverted_index_stream.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -373,12 +373,20 @@ func (it *InvertedIterator1) advanceInDb() {
if v, err = it.cursor.SeekBothRange(k, it.startTxKey[:]); err != nil {
panic(err)
}
if v != nil {
txNum := binary.BigEndian.Uint64(v)
if txNum < it.endTxNum {
it.nextDbKey = append(it.nextDbKey[:0], k...)
return
if v == nil {
seek, ok := kv.NextSubtree(k)
if !ok {
break
}
if k, _, err = it.cursor.Seek(seek); err != nil {
panic(err)
}
continue
}
txNum := binary.BigEndian.Uint64(v)
if txNum < it.endTxNum {
it.nextDbKey = append(it.nextDbKey[:0], k...)
return
}
if k, _, err = it.cursor.NextNoDup(); err != nil {
panic(err)
Expand Down
3 changes: 2 additions & 1 deletion erigon-lib/state/merge_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ package state

import (
"context"
"github.com/erigontech/erigon-lib/common/datadir"
"os"
"sort"
"testing"

"github.com/erigontech/erigon-lib/common/datadir"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
btree2 "github.com/tidwall/btree"
Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ replace (

require (
github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116
github.com/erigontech/mdbx-go v0.38.4
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55
github.com/erigontech/secp256k1 v1.1.0
github.com/erigontech/silkworm-go v0.18.0
)
Expand Down Expand Up @@ -186,7 +186,7 @@ require (
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect
github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect
github.com/imdario/mergo v0.3.11 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/ipfs/go-cid v0.4.1 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -269,8 +269,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp
github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M=
github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 h1:KCFa2uXEfZoBjV4buzjWmCmoqVLXiGCq0ZmQ2OjeRvQ=
github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116/go.mod h1:8vQ+VjvLu2gkPs8EwdPrOTAAo++WuLuBi54N7NuAF0I=
github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo=
github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI=
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4=
github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk=
github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY=
github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls=
github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA=
Expand Down Expand Up @@ -485,8 +485,8 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI
github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw=
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=
Expand Down

0 comments on commit 1347676

Please sign in to comment.