Skip to content

Commit

Permalink
Skip StorableNode/StorableTrie for new checkpoint
Browse files Browse the repository at this point in the history 
- Merge FlattenForest() with StoreCheckpoint() to iterate and serialize
nodes without creating intermediate StorableNode/StorableTrie objects.

- Stream encode nodes to avoid creating 400+ million element slice
  holding all nodes.

- Change checkpoint file format (v4) to store node count and trie count
at the footer (instead of header) required for stream encoding.

- Support previous checkpoint formats (v1, v3).
  • Loading branch information
@fxamacker
fxamacker committed Feb 2, 2022
1 parent dfafbd0 commit f185fd9
Show file tree
Hide file tree
Showing 7 changed files with 469 additions and 98 deletions.
8 changes: 1 addition & 7 deletions ledger/complete/ledger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/onflow/flow-go/ledger/common/hash"
"github.com/onflow/flow-go/ledger/common/pathfinder"
"github.com/onflow/flow-go/ledger/complete/mtrie"
"github.com/onflow/flow-go/ledger/complete/mtrie/flattener"
"github.com/onflow/flow-go/ledger/complete/mtrie/trie"
"github.com/onflow/flow-go/ledger/complete/wal"
"github.com/onflow/flow-go/module"
Expand Down Expand Up @@ -359,14 +358,9 @@ func (l *Ledger) ExportCheckpointAt(
return ledger.State(hash.DummyHash), fmt.Errorf("failed to create a checkpoint writer: %w", err)
}

flatTrie, err := flattener.FlattenTrie(newTrie)
if err != nil {
return ledger.State(hash.DummyHash), fmt.Errorf("failed to flatten the trie: %w", err)
}

l.logger.Info().Msg("storing the checkpoint to the file")

err = wal.StoreCheckpoint(flatTrie.ToFlattenedForestWithASingleTrie(), writer)
err = wal.StoreCheckpoint(writer, newTrie)
if err != nil {
return ledger.State(hash.DummyHash), fmt.Errorf("failed to store the checkpoint: %w", err)
}
Expand Down
56 changes: 40 additions & 16 deletions ledger/complete/mtrie/flattener/encoding.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,42 +4,57 @@ import (
"fmt"
"io"

"github.com/onflow/flow-go/ledger"
"github.com/onflow/flow-go/ledger/common/encoding"
"github.com/onflow/flow-go/ledger/common/utils"
"github.com/onflow/flow-go/ledger/complete/mtrie/node"
"github.com/onflow/flow-go/ledger/complete/mtrie/trie"
)

const encodingDecodingVersion = uint16(0)

// EncodeStorableNode encodes StorableNode
func EncodeStorableNode(storableNode *StorableNode) []byte {
// EncodeNode encodes node.
// TODO: reuse buffer
func EncodeNode(n *node.Node, lchildIndex uint64, rchildIndex uint64) []byte {

encPayload := encoding.EncodePayload(n.Payload())

length := 2 + 2 + 8 + 8 + 2 + 8 + 2 + len(n.Path()) + 4 + len(encPayload) + 2 + len(n.Hash())

length := 2 + 2 + 8 + 8 + 2 + 8 + 2 + len(storableNode.Path) + 4 + len(storableNode.EncPayload) + 2 + len(storableNode.HashValue)
buf := make([]byte, 0, length)

// 2-bytes encoding version
buf = utils.AppendUint16(buf, encodingDecodingVersion)

// 2-bytes Big Endian uint16 height
buf = utils.AppendUint16(buf, storableNode.Height)
buf = utils.AppendUint16(buf, uint16(n.Height()))

// 8-bytes Big Endian uint64 LIndex
buf = utils.AppendUint64(buf, storableNode.LIndex)
buf = utils.AppendUint64(buf, lchildIndex)

// 8-bytes Big Endian uint64 RIndex
buf = utils.AppendUint64(buf, storableNode.RIndex)
buf = utils.AppendUint64(buf, rchildIndex)

// 2-bytes Big Endian maxDepth
buf = utils.AppendUint16(buf, storableNode.MaxDepth)
buf = utils.AppendUint16(buf, n.MaxDepth())

// 8-bytes Big Endian regCount
buf = utils.AppendUint64(buf, storableNode.RegCount)
buf = utils.AppendUint64(buf, n.RegCount())

// 2-bytes Big Endian uint16 encoded path length and n-bytes encoded path
buf = utils.AppendShortData(buf, storableNode.Path)
path := n.Path()
if path != nil {
buf = utils.AppendShortData(buf, path[:])
} else {
buf = utils.AppendShortData(buf, nil)
}

// 4-bytes Big Endian uint32 encoded payload length and n-bytes encoded payload
buf = utils.AppendLongData(buf, storableNode.EncPayload)
buf = utils.AppendLongData(buf, encPayload)

// 2-bytes Big Endian uint16 hashValue length and n-bytes hashValue
buf = utils.AppendShortData(buf, storableNode.HashValue)
hash := n.Hash()
buf = utils.AppendShortData(buf, hash[:])

return buf
}
Expand Down Expand Up @@ -122,18 +137,27 @@ func ReadStorableNode(reader io.Reader) (*StorableNode, error) {
return storableNode, nil
}

// EncodeStorableTrie encodes StorableTrie
func EncodeStorableTrie(storableTrie *StorableTrie) []byte {
length := 2 + 8 + 2 + len(storableTrie.RootHash)
// EncodeTrie encodes trie root node
// TODO: reuse buffer
func EncodeTrie(rootNode *node.Node, rootIndex uint64) []byte {
// Get root hash
var rootHash ledger.RootHash
if rootNode == nil {
rootHash = trie.EmptyTrieRootHash()
} else {
rootHash = ledger.RootHash(rootNode.Hash())
}

length := 2 + 8 + 2 + len(rootHash)
buf := make([]byte, 0, length)
// 2-bytes encoding version
buf = utils.AppendUint16(buf, encodingDecodingVersion)

// 8-bytes Big Endian uint64 RootIndex
buf = utils.AppendUint64(buf, storableTrie.RootIndex)
buf = utils.AppendUint64(buf, rootIndex)

// 2-bytes Big Endian uint16 RootHash length and n-bytes RootHash
buf = utils.AppendShortData(buf, storableTrie.RootHash)
buf = utils.AppendShortData(buf, rootHash[:])

return buf
}
Expand Down
33 changes: 25 additions & 8 deletions ledger/complete/mtrie/flattener/encoding_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,21 +7,28 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/onflow/flow-go/ledger"
"github.com/onflow/flow-go/ledger/common/encoding"
"github.com/onflow/flow-go/ledger/common/hash"
"github.com/onflow/flow-go/ledger/common/utils"
"github.com/onflow/flow-go/ledger/complete/mtrie/flattener"
"github.com/onflow/flow-go/ledger/complete/mtrie/node"
)

func TestStorableNode(t *testing.T) {
path := utils.PathByUint8(3)
payload := utils.LightPayload8('A', 'a')
hashValue := hash.Hash([32]byte{4, 4, 4})

n := node.NewNode(2137, nil, nil, path, payload, hashValue, 7, 5000)

storableNode := &flattener.StorableNode{
LIndex: 1,
RIndex: 2,
Height: 2137,
Path: path[:],
EncPayload: encoding.EncodePayload(utils.LightPayload8('A', 'a')),
HashValue: []byte{4, 4, 4},
EncPayload: encoding.EncodePayload(payload),
HashValue: hashValue[:],
MaxDepth: 7,
RegCount: 5000,
}
Expand All @@ -39,12 +46,15 @@ func TestStorableNode(t *testing.T) {
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // path data
0, 0, 0, 25, // payload data len
0, 0, 6, 0, 0, 0, 9, 0, 1, 0, 0, 0, 3, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 1, 97, // payload data
0, 3, // hashValue length
4, 4, 4, // hashValue
0, 32, // hashValue length
4, 4, 4, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, // hashValue
}

t.Run("encode", func(t *testing.T) {
data := flattener.EncodeStorableNode(storableNode)
data := flattener.EncodeNode(n, 1, 2)
assert.Equal(t, expected, data)
})

Expand All @@ -57,21 +67,28 @@ func TestStorableNode(t *testing.T) {
}

func TestStorableTrie(t *testing.T) {
hashValue := hash.Hash([32]byte{2, 2, 2})

rootNode := node.NewNode(256, nil, nil, ledger.DummyPath, nil, hashValue, 7, 5000)

storableTrie := &flattener.StorableTrie{
RootIndex: 21,
RootHash: []byte{2, 2, 2},
RootHash: hashValue[:],
}

// Version 0
expected := []byte{
0, 0, // encoding version
0, 0, 0, 0, 0, 0, 0, 21, // RootIndex
0, 3, 2, 2, 2, // RootHash length + data
0, 32, // RootHash length
2, 2, 2, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, // RootHash data
}

t.Run("encode", func(t *testing.T) {
data := flattener.EncodeStorableTrie(storableTrie)
data := flattener.EncodeTrie(rootNode, 21)

assert.Equal(t, expected, data)
})
Expand Down
Loading

0 comments on commit f185fd9

Please sign in to comment.