diff --git a/Makefile b/Makefile index c78ffa8530d..6a7fbe8a17e 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ GOPRIVATE = github.com/erigontech/silkworm-go PACKAGE = github.com/erigontech/erigon -GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false +GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}" GOBUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build $(GO_FLAGS) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index c1b5175cb46..6a4d5bdb537 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -21,10 +21,10 @@ package rawdb import ( "encoding/binary" + libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon/core/types" ) diff --git a/core/state/intra_block_state_logger_test.go b/core/state/intra_block_state_logger_test.go index cc5fed21804..64cc73d59f6 100644 --- a/core/state/intra_block_state_logger_test.go +++ b/core/state/intra_block_state_logger_test.go @@ -20,17 +20,16 @@ import ( "reflect" "testing" + "github.com/holiman/uint256" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + libcommon "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/kv/rawdbv3" "github.com/erigontech/erigon-lib/log/v3" stateLib "github.com/erigontech/erigon-lib/state" - "github.com/erigontech/erigon/core/tracing" "github.com/erigontech/erigon/core/tracing/mocks" - - "github.com/holiman/uint256" - "github.com/stretchr/testify/require" - gomock "go.uber.org/mock/gomock" ) func TestStateLogger(t *testing.T) { diff --git a/erigon-lib/config3/config3.go b/erigon-lib/config3/config3.go index be0b43b3d98..8c2c07bdb92 100644 --- a/erigon-lib/config3/config3.go +++ b/erigon-lib/config3/config3.go @@ -18,6 +18,7 @@ package config3 // Default number of transactions (txNums) in one "step". One static file can have [1, 64] steps. const DefaultStepSize = 1_562_500 // = 100M / 64. Dividers: 2, 5, 10, 20, 50, 100, 500 +//const DefaultStepSize = 1_562_500 / 10 // StepsInFrozenFile - files of this size are completely frozen/immutable. // files of smaller size are also immutable, but can be removed after merge to bigger files. diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index b46410c6a3a..cece7b1848c 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -10,7 +10,7 @@ replace ( require ( github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed - github.com/erigontech/mdbx-go v0.38.4 + github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 github.com/erigontech/secp256k1 v1.1.0 github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 ) @@ -62,7 +62,7 @@ require ( require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect + github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 65818353ba7..b08794204eb 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -154,8 +154,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed h1:un44S8Tuol4LBIC6R94t93GShM53BYjz7GsNPziDLQ8= github.com/erigontech/interfaces v0.0.0-20241120074553-214b5fd396ed/go.mod h1:N7OUkhkcagp9+7yb4ycHsG2VWCOmuJ1ONBecJshxtLE= -github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= -github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U= @@ -271,8 +271,8 @@ github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= diff --git a/erigon-lib/kv/mdbx/kv_mdbx.go b/erigon-lib/kv/mdbx/kv_mdbx.go index c70c671f0e9..b7a52c6d06b 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx.go +++ b/erigon-lib/kv/mdbx/kv_mdbx.go @@ -202,7 +202,7 @@ func (opts MdbxOpts) Open(ctx context.Context) (kv.RwDB, error) { } - env, err := mdbx.NewEnv() + env, err := mdbx.NewEnv(mdbx.Label(opts.label)) if err != nil { return nil, err } diff --git a/erigon-lib/kv/mdbx/kv_mdbx_test.go b/erigon-lib/kv/mdbx/kv_mdbx_test.go index 5b0d7fa9b7f..fad3e08d525 100644 --- a/erigon-lib/kv/mdbx/kv_mdbx_test.go +++ b/erigon-lib/kv/mdbx/kv_mdbx_test.go @@ -458,64 +458,6 @@ func TestAppendFirstLast(t *testing.T) { require.Equal(t, []string{"value6.1"}, values) } -func TestNextPrevCurrent(t *testing.T) { - _, _, c := BaseCase(t) - - k, v, err := c.First() - require.Nil(t, err) - keys, values := iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Next() - require.Equal(t, []byte("key1"), k) - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - require.Equal(t, k, []byte("key1")) - require.Equal(t, v, []byte("value1.3")) - - k, v, err = c.Next() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key3", "key3"}, keys) - require.Equal(t, []string{"value3.1", "value3.3"}, values) - - k, v, err = c.Prev() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - - k, v, err = c.Prev() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.1", "value1.3", "value3.1", "value3.3"}, values) - - err = c.DeleteCurrent() - require.Nil(t, err) - k, v, err = c.Current() - require.Nil(t, err) - keys, values = iteration(t, c, k, v) - require.Equal(t, []string{"key1", "key3", "key3"}, keys) - require.Equal(t, []string{"value1.3", "value3.1", "value3.3"}, values) - -} - func TestSeek(t *testing.T) { _, _, c := BaseCase(t) diff --git a/erigon-lib/state/history_stream.go b/erigon-lib/state/history_stream.go index 7f22adc7b20..083eaccefc0 100644 --- a/erigon-lib/state/history_stream.go +++ b/erigon-lib/state/history_stream.go @@ -283,10 +283,11 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error { } seek = next } - for k, _, err := hi.valsCDup.Seek(seek); k != nil; k, _, err = hi.valsCDup.NextNoDup() { - if err != nil { - return err - } + k, _, err := hi.valsCDup.Seek(seek) + if err != nil { + return err + } + for k != nil { if hi.toPrefix != nil && bytes.Compare(k, hi.toPrefix) >= 0 { break } @@ -295,8 +296,16 @@ func (hi *HistoryRangeAsOfDB) advanceSmallVals() error { return err } if v == nil { + seek, ok := kv.NextSubtree(k) + if !ok { + break + } + if k, _, err = hi.valsCDup.Seek(seek); err != nil { + panic(err) + } continue } + hi.nextKey = k hi.nextVal = v[8:] return nil @@ -566,24 +575,33 @@ func (hi *HistoryChangesIterDB) advanceSmallVals() (err error) { return err } } - for ; k != nil; k, _, err = hi.valsCDup.NextNoDup() { - if err != nil { - return err - } + for k != nil { v, err := hi.valsCDup.SeekBothRange(k, hi.startTxKey[:]) if err != nil { return err } if v == nil { + next, ok := kv.NextSubtree(k) + if !ok { + hi.nextKey = nil + return nil + } + k, _, err = hi.valsCDup.Seek(next) + if err != nil { + return err + } continue } foundTxNumVal := v[:8] - if hi.endTxNum >= 0 && int(binary.BigEndian.Uint64(foundTxNumVal)) >= hi.endTxNum { - continue + if hi.endTxNum < 0 || int(binary.BigEndian.Uint64(foundTxNumVal)) < hi.endTxNum { + hi.nextKey = k + hi.nextVal = v[8:] + return nil + } + k, _, err = hi.valsCDup.NextNoDup() + if err != nil { + return err } - hi.nextKey = k - hi.nextVal = v[8:] - return nil } hi.nextKey = nil return nil diff --git a/erigon-lib/state/history_test.go b/erigon-lib/state/history_test.go index ba5df93e6f8..d8462ca7e14 100644 --- a/erigon-lib/state/history_test.go +++ b/erigon-lib/state/history_test.go @@ -28,25 +28,23 @@ import ( "testing" "time" - "github.com/erigontech/erigon-lib/common/length" - "github.com/erigontech/erigon-lib/config3" - "github.com/erigontech/erigon-lib/log/v3" - "github.com/erigontech/erigon-lib/seg" - - "github.com/erigontech/erigon-lib/common" - "github.com/erigontech/erigon-lib/common/datadir" - "github.com/stretchr/testify/require" btree2 "github.com/tidwall/btree" + "github.com/erigontech/erigon-lib/common" "github.com/erigontech/erigon-lib/common/background" + "github.com/erigontech/erigon-lib/common/datadir" "github.com/erigontech/erigon-lib/common/hexutility" + "github.com/erigontech/erigon-lib/common/length" + "github.com/erigontech/erigon-lib/config3" "github.com/erigontech/erigon-lib/kv" "github.com/erigontech/erigon-lib/kv/mdbx" "github.com/erigontech/erigon-lib/kv/order" "github.com/erigontech/erigon-lib/kv/stream" + "github.com/erigontech/erigon-lib/log/v3" "github.com/erigontech/erigon-lib/recsplit" "github.com/erigontech/erigon-lib/recsplit/eliasfano32" + "github.com/erigontech/erigon-lib/seg" ) func testDbAndHistory(tb testing.TB, largeValues bool, logger log.Logger) (kv.RwDB, *History) { diff --git a/erigon-lib/state/inverted_index_stream.go b/erigon-lib/state/inverted_index_stream.go index 5f7b5808f16..191b38ea862 100644 --- a/erigon-lib/state/inverted_index_stream.go +++ b/erigon-lib/state/inverted_index_stream.go @@ -373,12 +373,20 @@ func (it *InvertedIterator1) advanceInDb() { if v, err = it.cursor.SeekBothRange(k, it.startTxKey[:]); err != nil { panic(err) } - if v != nil { - txNum := binary.BigEndian.Uint64(v) - if txNum < it.endTxNum { - it.nextDbKey = append(it.nextDbKey[:0], k...) - return + if v == nil { + seek, ok := kv.NextSubtree(k) + if !ok { + break } + if k, _, err = it.cursor.Seek(seek); err != nil { + panic(err) + } + continue + } + txNum := binary.BigEndian.Uint64(v) + if txNum < it.endTxNum { + it.nextDbKey = append(it.nextDbKey[:0], k...) + return } if k, _, err = it.cursor.NextNoDup(); err != nil { panic(err) diff --git a/erigon-lib/state/merge_test.go b/erigon-lib/state/merge_test.go index c7fa5124e06..3838440dec1 100644 --- a/erigon-lib/state/merge_test.go +++ b/erigon-lib/state/merge_test.go @@ -18,11 +18,12 @@ package state import ( "context" - "github.com/erigontech/erigon-lib/common/datadir" "os" "sort" "testing" + "github.com/erigontech/erigon-lib/common/datadir" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" btree2 "github.com/tidwall/btree" diff --git a/go.mod b/go.mod index b1d51ea37a2..721f6a8a9a2 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ replace ( require ( github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 - github.com/erigontech/mdbx-go v0.38.4 + github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 github.com/erigontech/secp256k1 v1.1.0 github.com/erigontech/silkworm-go v0.18.0 ) @@ -186,7 +186,7 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20241017200806-017d972448fc // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect - github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect + github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect diff --git a/go.sum b/go.sum index ff4b776ceda..d271898f3df 100644 --- a/go.sum +++ b/go.sum @@ -269,8 +269,8 @@ github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e h1:Zp github.com/erigontech/erigon-snapshot v1.3.1-0.20241023024258-f64407a77e8e/go.mod h1:ooHlCl+eEYzebiPu+FP6Q6SpPUeMADn8Jxabv3IKb9M= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116 h1:KCFa2uXEfZoBjV4buzjWmCmoqVLXiGCq0ZmQ2OjeRvQ= github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116/go.mod h1:8vQ+VjvLu2gkPs8EwdPrOTAAo++WuLuBi54N7NuAF0I= -github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= -github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55 h1:OOIbmoNOak87yBBBw6MNmkTZi+A76Rof/ZfWVXbOj+4= +github.com/erigontech/mdbx-go v0.39.0-alpha.0.20241223021833-1b75fb145a55/go.mod h1:ncKVXcwnMpZ+wIye89HLVglDwXFXbEY2L1OI1Iahgzk= github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY= github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= @@ -485,8 +485,8 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d h1:Azx2B59D4+zpVVtuYb8Oe3uOLi/ift4xfwKdhBX0Cy0= -github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e h1:8AnObPi8WmIgjwcidUxaREhXMSpyUJeeSrIkZTXdabw= +github.com/ianlancetaylor/cgosymbolizer v0.0.0-20241129212102-9c50ad6b591e/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA=