Added reverse beacon changeset for beacon state rewind (erigontech#7185)

Added changesets for beacon chain to implement memory efficient fork
choice

cl/cltypes/attestations.go

@@ -531,7 +531,6 @@ func (a *PendingAttestation) EncodeSSZ(buf []byte) (dst []byte, err error) {
 	if dst, err = a.Data.EncodeSSZ(dst); err != nil {
 		return
 	}
-	fmt.Println(dst)
 	dst = append(dst, ssz.Uint64SSZ(a.InclusionDelay)...)
 	dst = append(dst, ssz.Uint64SSZ(a.ProposerIndex)...)
 

cl/cltypes/justification_bits.go

@@ -33,3 +33,7 @@ func (j JustificationBits) CheckRange(start int, end int) bool {
 	}
 	return true
 }
+
+func (j JustificationBits) Copy() JustificationBits {
+	return JustificationBits{j[0], j[1], j[2], j[3]}
+}

cl/cltypes/participation_flags.go

@@ -25,6 +25,12 @@ func (p ParticipationFlagsList) Bytes() []byte {
 	return b
 }
 
+func (p ParticipationFlagsList) Copy() ParticipationFlagsList {
+	c := make(ParticipationFlagsList, len(p))
+	copy(c, p)
+	return c
+}
+
 func ParticipationFlagsListFromBytes(buf []byte) ParticipationFlagsList {
 	flagsList := make([]ParticipationFlags, len(buf))
 	for i := range flagsList {

cmd/ef-tests-cl/consensus_tests/fork.go

@@ -20,6 +20,7 @@ func forkTest(context testContext) error {
 		return err
 	}
 
+	preState.StartCollectingReverseChangeSet()
 	if preState.Version() == clparams.Phase0Version {
 		if err := preState.UpgradeToAltair(); err != nil {
 			return err
@@ -33,6 +34,8 @@ func forkTest(context testContext) error {
 			return err
 		}
 	}
+	change := preState.StopCollectingReverseChangeSet()
+
 	if expectedError {
 		return fmt.Errorf("expected error")
 	}
@@ -47,5 +50,25 @@ func forkTest(context testContext) error {
 	if root != expectedRoot {
 		return fmt.Errorf("mismatching state roots")
 	}
+	if context.version == clparams.AltairVersion {
+		return nil
+	}
+	// now do unwind
+	initialState, err := decodeStateFromFile(prevContext, "pre.ssz_snappy")
+	if err != nil {
+		return err
+	}
+	preState.RevertWithChangeset(change)
+	root, err = preState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	expectedRoot, err = initialState.HashSSZ()
+	if err != nil {
+		return err
+	}
+	if root != expectedRoot {
+		return fmt.Errorf("mismatching state roots with unwind")
+	}
 	return nil
 }

cmd/ef-tests-cl/consensus_tests/sanity.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/beacon_changeset"
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/transition"
 )
 
@@ -28,12 +30,16 @@ func testSanityFunction(context testContext) error {
 		return err
 	}
 	startSlot := testState.Slot()
+
+	changes := []*beacon_changeset.ReverseBeaconStateChangeSet{}
 	var block *cltypes.SignedBeaconBlock
 	for _, block = range blocks {
+		testState.StartCollectingReverseChangeSet()
 		err = transition.TransitionState(testState, block, true)
 		if err != nil {
 			break
 		}
+		changes = append(changes, testState.StopCollectingReverseChangeSet())
 	}
 	// Deal with transition error
 	if expectedError && err == nil {
@@ -45,17 +51,55 @@ func testSanityFunction(context testContext) error {
 		}
 		return fmt.Errorf("cannot transition state: %s. slot=%d. start_slot=%d", err, block.Block.Slot, startSlot)
 	}
-	expectedRoot, err := expectedState.HashSSZ()
+	finalRoot, err := expectedState.HashSSZ()
 	if err != nil {
 		return err
 	}
 	haveRoot, err := testState.HashSSZ()
 	if err != nil {
 		return err
 	}
-	if haveRoot != expectedRoot {
+	if haveRoot != finalRoot {
 		return fmt.Errorf("mismatching state roots")
 	}
+	if context.version == clparams.Phase0Version {
+		return nil
+	}
+	// Now do the unwind
+	initialState, err := decodeStateFromFile(context, "pre.ssz_snappy")
+	if err != nil {
+		return err
+	}
+	_ = initialState
+	for i := len(changes) - 1; i >= 0; i-- {
+		testState.RevertWithChangeset(changes[i])
+	}
+
+	expectedRoot, err := initialState.HashSSZ()
+	if err != nil {
+		return err
+	}
+
+	haveRoot, err = testState.HashSSZ()
+	if err != nil {
+		return err
+	}
+
+	if haveRoot != expectedRoot {
+		return fmt.Errorf("mismatching state roots with unwind")
+	}
+	// Execute them back (ensure cache is good.)
+	for _, block = range blocks {
+		testState.StartCollectingReverseChangeSet()
+		err = transition.TransitionState(testState, block, true)
+		if err != nil {
+			break
+		}
+		changes = append(changes, testState.StopCollectingReverseChangeSet())
+	}
+	if err != nil {
+		return err
+	}
 	return nil
 }
 

cmd/erigon-cl/core/beacon_changeset/list_changeset.go

@@ -0,0 +1,146 @@
+package beacon_changeset
+
+import (
+	"sort"
+)
+
+type ListChangeSet[T any] struct {
+	list       []*listElementChangeset[T]
+	listLength int
+	nextId     int
+	compact    bool
+}
+
+type listElementChangeset[T any] struct {
+	value     T
+	listIndex int
+	id        int
+}
+
+// NewListChangeSet creates new list with given length.
+func NewListChangeSet[T any](length int) *ListChangeSet[T] {
+	return &ListChangeSet[T]{listLength: length}
+}
+
+// AddChange appens to a new change to the changeset of the list.
+func (l *ListChangeSet[T]) AddChange(index int, elem T) {
+	l.compact = false
+	l.list = append(l.list, &listElementChangeset[T]{
+		value:     elem,
+		listIndex: index,
+		id:        l.nextId,
+	})
+	l.nextId++
+}
+
+// CompactChanges removes duplicates from a list using QuickSort and linear scan.
+// duplicates may appear if one state parameter is changed more than once.
+func (l *ListChangeSet[T]) CompactChanges() {
+	if l.compact {
+		return
+	}
+	l.compact = true
+	// Check if there are any duplicates to remove.
+	if len(l.list) < 2 {
+		return
+	}
+
+	// Sort the list using QuickSort.
+	sort.Slice(l.list, func(i, j int) bool {
+		if l.list[i].listIndex == l.list[j].listIndex {
+			return l.list[i].id < l.list[j].id
+		}
+		return l.list[i].listIndex < l.list[j].listIndex
+	})
+
+	// Create a new list buffer for the compacted list.
+	compactList := []*listElementChangeset[T]{}
+
+	// Do a linear scan through the sorted list and remove duplicates.
+	previousIndexElement := l.list[0]
+	for _, listElement := range l.list {
+		if listElement.listIndex != previousIndexElement.listIndex {
+			compactList = append(compactList, previousIndexElement)
+		}
+		previousIndexElement = listElement
+	}
+	compactList = append(compactList, previousIndexElement)
+
+	// Update the original list with the compacted list.
+	l.list = compactList
+}
+
+// CompactChangesReverse removes duplicates from a list using QuickSort and linear scan.
+// duplicates may appear if one state parameter is changed more than once.
+// Difference with CompactChanges is that the sorting is reversed.
+func (l *ListChangeSet[T]) CompactChangesReverse() {
+	if l.compact {
+		return
+	}
+	l.compact = true
+	// Check if there are any duplicates to remove.
+	if len(l.list) < 2 {
+		return
+	}
+
+	// Sort the list using QuickSort.
+	sort.Slice(l.list, func(i, j int) bool {
+		if l.list[i].listIndex == l.list[j].listIndex {
+			return l.list[i].id > l.list[j].id
+		}
+		return l.list[i].listIndex < l.list[j].listIndex
+	})
+
+	// Create a new list buffer for the compacted list.
+	compactList := []*listElementChangeset[T]{}
+
+	// Do a linear scan through the sorted list and remove duplicates.
+	previousIndexElement := l.list[0]
+	for _, listElement := range l.list {
+		if listElement.listIndex != previousIndexElement.listIndex {
+			compactList = append(compactList, previousIndexElement)
+		}
+		previousIndexElement = listElement
+	}
+	compactList = append(compactList, previousIndexElement)
+	// Update the original list with the compacted list.
+	l.list = compactList
+}
+
+// ApplyChanges Apply changes without any mercy. if it is reverse, you need to call CompactChangesReverse before.
+func (l *ListChangeSet[T]) ApplyChanges(input []T) (output []T, changed bool) {
+	if len(l.list) == 0 && l.listLength == len(input) {
+		output = input
+		return
+	}
+	changed = true
+	// Re-adjust list size.
+	output = make([]T, l.listLength)
+	copy(output, input)
+	// Now apply changes to the given list
+	for _, elem := range l.list {
+		if elem.listIndex >= len(output) {
+			continue
+		}
+		output[elem.listIndex] = elem.value
+	}
+	return
+}
+
+// ChangesWithHandler uses custom handler to handle changes.
+func (l *ListChangeSet[T]) ChangesWithHandler(fn func(value T, index int)) {
+	// Now apply changes to the given list
+	for _, elem := range l.list {
+		fn(elem.value, elem.listIndex)
+	}
+}
+
+// ListLength return full list length
+func (l *ListChangeSet[T]) ListLength() int {
+	return l.listLength
+}
+
+// Empty return whether current list diff is empty
+func (l *ListChangeSet[T]) Empty() bool {
+	return len(l.list) == 0
+}

cmd/erigon-cl/core/beacon_changeset/list_changeset_test.go

@@ -0,0 +1,45 @@
+package beacon_changeset
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestListChangeset(t *testing.T) {
+	pre := []int{6, 8, 9}
+	changeset := NewListChangeSet[int](3)
+	changeset.AddChange(1, 45)
+	changeset.AddChange(1, 1)
+	changeset.AddChange(2, 45)
+	changeset.CompactChanges()
+	require.Equal(t, len(changeset.list), 2)
+	post, changed := changeset.ApplyChanges(pre)
+	require.Equal(t, post, []int{6, 1, 45})
+	require.Equal(t, changed, true)
+}
+
+func TestListChangesetWithReverse(t *testing.T) {
+	pre := []int{6, 8, 9}
+	changeset := NewListChangeSet[int](3)
+	changeset.AddChange(1, 45)
+	changeset.AddChange(1, 1)
+	changeset.AddChange(2, 45)
+	changeset.CompactChangesReverse()
+	require.Equal(t, len(changeset.list), 2)
+	post, changed := changeset.ApplyChanges(pre)
+	require.Equal(t, post, []int{6, 45, 45})
+	require.Equal(t, changed, true)
+}
+
+func TestListChangesetWithoutCompact(t *testing.T) {
+	pre := []int{6, 8, 9}
+	changeset := NewListChangeSet[int](3)
+	changeset.AddChange(1, 45)
+	changeset.AddChange(1, 1)
+	changeset.AddChange(2, 45)
+	require.Equal(t, len(changeset.list), 3)
+	post, changed := changeset.ApplyChanges(pre)
+	require.Equal(t, post, []int{6, 1, 45})
+	require.Equal(t, changed, true)
+}