Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(trie): Implement parity trie crate #3390

Closed
wants to merge 47 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
47 commits
Select commit Hold shift + click to select a range
5ecc13a
feat(trie): Add trie v1 new headers support
dimartiro May 24, 2023
b2b3944
Refa: rename variants following substrate names
dimartiro Jun 1, 2023
f4ef2a9
feat(trie): Decode nodes with hashed values
dimartiro Jun 7, 2023
7f72f43
Add todo reminders
dimartiro Jun 7, 2023
894d9b7
feat(trie): Encode new node headers
dimartiro Jun 7, 2023
4ff225a
feat(trie): Add encode node with hashed values impl
dimartiro Jun 7, 2023
43f4f41
fix(trie): Decode hashed value
dimartiro Jun 7, 2023
e0f7571
feat(trie): Add short hashed values checks
dimartiro Jun 12, 2023
819792b
feat(trie): Add encode tests
dimartiro Jun 14, 2023
e7b4360
feat(trie): One more test to cover all lines
dimartiro Jun 14, 2023
7dc48b7
fix(trie): Improve error messages
dimartiro Jun 21, 2023
813413f
fix(trie/proof): Get value from hashed node
dimartiro Jul 3, 2023
d415a7e
fix tests
dimartiro Jul 4, 2023
ada5ad2
fix linting
dimartiro Jul 4, 2023
b69428d
feat(trie/proof): Use predifined nodes DB
dimartiro Jul 4, 2023
ce19d2c
feat(trie/proof): Add copyright
dimartiro Jul 4, 2023
9b17564
fix(trie/proof): tests
dimartiro Jul 4, 2023
628aa4e
fix(trie/proof): Fix verify proof
dimartiro Jul 4, 2023
78106c5
fix(trie/proof): Add mock
dimartiro Jul 4, 2023
9376bf7
Fix mock and add copyright
dimartiro Jul 4, 2023
2f0bbd4
Adds copyright
dimartiro Jul 4, 2023
c285da9
Fix type error
dimartiro Jul 4, 2023
6ce3e01
Remove use of experiment libs
dimartiro Jul 6, 2023
d66ff06
PR comments
dimartiro Jul 6, 2023
85f7523
Idiomatic comments
dimartiro Jul 6, 2023
d1441b7
Use empty variant bits instead plain bits
dimartiro Jul 16, 2023
9fb4d9c
Mock trie db
dimartiro Jul 16, 2023
98a87cf
fix(trie): Fix node copy & improve test coverage
dimartiro Jul 17, 2023
7ffd9e5
feat(trie): TrieDB and lookup refactor
dimartiro Jul 17, 2023
0390ade
Change constant
dimartiro Jul 20, 2023
1ba7e9c
feat(trie): Add new lookup method
dimartiro Jul 26, 2023
c2ad381
Merge remote-tracking branch 'origin/development' into diego/trieV1/r…
dimartiro Jul 26, 2023
9ab4b48
fix(trie): interfaces conflict
dimartiro Jul 27, 2023
d31cad7
Merge remote-tracking branch 'origin/development' into diego/trieV1/r…
dimartiro Jul 27, 2023
36251ef
fix(trie): Add missing copyrights
dimartiro Jul 27, 2023
ac475c7
fix(trie): Linting
dimartiro Jul 27, 2023
de4c7eb
fix(trie): Fix key conversion
dimartiro Jul 27, 2023
afcc752
fix(trie): Refactor
dimartiro Jul 28, 2023
0f892c8
fix(trie): Linting
dimartiro Jul 28, 2023
fc531c6
feat(trie): Add lookup with lazy loading
dimartiro Jul 29, 2023
a8ee36e
Resolve deepsource checks
dimartiro Jul 29, 2023
42fc93f
Removes trielayout
dimartiro Aug 2, 2023
5440eeb
Merge remote-tracking branch 'origin/development' into diego/trieV1/r…
dimartiro Aug 2, 2023
e284a01
Unexport unnecesary exported functions
dimartiro Aug 2, 2023
1e48ee3
refactor(trie): Move to pkg folder
dimartiro Aug 4, 2023
b443cd1
Merge remote-tracking branch 'origin/development' into diego/trieV1/r…
dimartiro Aug 4, 2023
20a7ea7
Adds copyright
dimartiro Aug 4, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal/trie/node/children.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,7 @@ func (n *Node) HasChild() (has bool) {
}
return false
}

func (n *Node) ChildAt(i uint) *Node {
return n.Children[i]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't we do a length check here?

}
26 changes: 14 additions & 12 deletions internal/trie/node/decode.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ var (
ErrDecodeChildHash = errors.New("cannot decode child hash")
)

var EmptyNode = &Node{}

const hashLength = common.HashLength

// Decode decodes a node from a reader.
Expand All @@ -34,21 +36,21 @@ const hashLength = common.HashLength
// For branch decoding, see the comments on decodeBranch.
// For leaf decoding, see the comments on decodeLeaf.
func Decode(reader io.Reader) (n *Node, err error) {
variant, partialKeyLength, err := decodeHeader(reader)
variant, partialKeyLength, err := DecodeHeader(reader)
if err != nil {
return nil, fmt.Errorf("decoding header: %w", err)
}

switch variant {
case emptyVariant:
return nil, nil //nolint:nilnil
case leafVariant, leafWithHashedValueVariant:
case EmptyVariant:
return EmptyNode, nil
case LeafVariant, LeafWithHashedValueVariant:
n, err = decodeLeaf(reader, variant, partialKeyLength)
if err != nil {
return nil, fmt.Errorf("cannot decode leaf: %w", err)
}
return n, nil
case branchVariant, branchWithValueVariant, branchWithHashedValueVariant:
case BranchVariant, BranchWithValueVariant, BranchWithHashedValueVariant:
n, err = decodeBranch(reader, variant, partialKeyLength)
if err != nil {
return nil, fmt.Errorf("cannot decode branch: %w", err)
Expand All @@ -65,13 +67,13 @@ func Decode(reader io.Reader) (n *Node, err error) {
// reconstructing the child nodes from the encoding. This function instead stubs where the
// children are known to be with an empty leaf. The children nodes hashes are then used to
// find other storage values using the persistent database.
func decodeBranch(reader io.Reader, variant variant, partialKeyLength uint16) (
func decodeBranch(reader io.Reader, variant Variant, partialKeyLength uint16) (
node *Node, err error) {
node = &Node{
Children: make([]*Node, ChildrenCapacity),
}

node.PartialKey, err = decodeKey(reader, partialKeyLength)
node.PartialKey, err = DecodeKey(reader, partialKeyLength)
if err != nil {
return nil, fmt.Errorf("cannot decode key: %w", err)
}
Expand All @@ -85,12 +87,12 @@ func decodeBranch(reader io.Reader, variant variant, partialKeyLength uint16) (
sd := scale.NewDecoder(reader)

switch variant {
case branchWithValueVariant:
case BranchWithValueVariant:
err := sd.Decode(&node.StorageValue)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrDecodeStorageValue, err)
}
case branchWithHashedValueVariant:
case BranchWithHashedValueVariant:
hashedValue, err := decodeHashedValue(reader)
if err != nil {
return nil, err
Expand Down Expand Up @@ -134,17 +136,17 @@ func decodeBranch(reader io.Reader, variant variant, partialKeyLength uint16) (
}

// decodeLeaf reads from a reader and decodes to a leaf node.
func decodeLeaf(reader io.Reader, variant variant, partialKeyLength uint16) (node *Node, err error) {
func decodeLeaf(reader io.Reader, variant Variant, partialKeyLength uint16) (node *Node, err error) {
node = &Node{}

node.PartialKey, err = decodeKey(reader, partialKeyLength)
node.PartialKey, err = DecodeKey(reader, partialKeyLength)
if err != nil {
return nil, fmt.Errorf("cannot decode key: %w", err)
}

sd := scale.NewDecoder(reader)

if variant == leafWithHashedValueVariant {
if variant == LeafWithHashedValueVariant {
hashedValue, err := decodeHashedValue(reader)
if err != nil {
return nil, err
Expand Down
56 changes: 28 additions & 28 deletions internal/trie/node/decode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,12 @@ func Test_Decode(t *testing.T) {
errMessage: "decoding header: decoding header byte: node variant is unknown: for header byte 00001000",
},
"empty_node": {
reader: bytes.NewReader([]byte{emptyVariant.bits}),
n: nil,
reader: bytes.NewReader([]byte{EmptyVariant.bits}),
n: EmptyNode,
},
"leaf_decoding_error": {
reader: bytes.NewReader([]byte{
leafVariant.bits | 1, // key length 1
LeafVariant.bits | 1, // key length 1
// missing key data byte
}),
errWrapped: io.EOF,
Expand All @@ -73,7 +73,7 @@ func Test_Decode(t *testing.T) {
},
"leaf_success": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{leafVariant.bits | 1}, // partial key length 1
{LeafVariant.bits | 1}, // partial key length 1
{9}, // key data
scaleEncodeBytes(t, 1, 2, 3),
})),
Expand All @@ -84,7 +84,7 @@ func Test_Decode(t *testing.T) {
},
"branch_decoding_error": {
reader: bytes.NewReader([]byte{
branchVariant.bits | 1, // key length 1
BranchVariant.bits | 1, // key length 1
// missing key data byte
}),
errWrapped: io.EOF,
Expand All @@ -93,7 +93,7 @@ func Test_Decode(t *testing.T) {
},
"branch_success": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{branchVariant.bits | 1}, // partial key length 1
{BranchVariant.bits | 1}, // partial key length 1
{9}, // key data
{0b0000_0000, 0b0000_0000}, // no children bitmap
})),
Expand All @@ -104,7 +104,7 @@ func Test_Decode(t *testing.T) {
},
"leaf_with_hashed_value_success": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{leafWithHashedValueVariant.bits | 1}, // partial key length 1
{LeafWithHashedValueVariant.bits | 1}, // partial key length 1
{9}, // key data
hashedValue.ToBytes(),
})),
Expand All @@ -116,7 +116,7 @@ func Test_Decode(t *testing.T) {
},
"leaf_with_hashed_value_fail_too_short": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{leafWithHashedValueVariant.bits | 1}, // partial key length 1
{LeafWithHashedValueVariant.bits | 1}, // partial key length 1
{9}, // key data
{0b0000_0000}, // less than 32bytes
})),
Expand All @@ -125,7 +125,7 @@ func Test_Decode(t *testing.T) {
},
"branch_with_hashed_value_success": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{branchWithHashedValueVariant.bits | 1}, // partial key length 1
{BranchWithHashedValueVariant.bits | 1}, // partial key length 1
{9}, // key data
{0b0000_0000, 0b0000_0000}, // no children bitmap
hashedValue.ToBytes(),
Expand All @@ -139,7 +139,7 @@ func Test_Decode(t *testing.T) {
},
"branch_with_hashed_value_fail_too_short": {
reader: bytes.NewReader(concatByteSlices([][]byte{
{branchWithHashedValueVariant.bits | 1}, // partial key length 1
{BranchWithHashedValueVariant.bits | 1}, // partial key length 1
{9}, // key data
{0b0000_0000, 0b0000_0000}, // no children bitmap
{0b0000_0000},
Expand Down Expand Up @@ -177,7 +177,7 @@ func Test_decodeBranch(t *testing.T) {

testCases := map[string]struct {
reader io.Reader
nodeVariant variant
nodeVariant Variant
partialKeyLength uint16
branch *Node
errWrapped error
Expand All @@ -187,7 +187,7 @@ func Test_decodeBranch(t *testing.T) {
reader: bytes.NewBuffer([]byte{
// missing key data byte
}),
nodeVariant: branchVariant,
nodeVariant: BranchVariant,
partialKeyLength: 1,
errWrapped: io.EOF,
errMessage: "cannot decode key: reading from reader: EOF",
Expand All @@ -197,7 +197,7 @@ func Test_decodeBranch(t *testing.T) {
9, // key data
// missing children bitmap 2 bytes
}),
nodeVariant: branchVariant,
nodeVariant: BranchVariant,
partialKeyLength: 1,
errWrapped: ErrReadChildrenBitmap,
errMessage: "cannot read children bitmap: EOF",
Expand All @@ -208,7 +208,7 @@ func Test_decodeBranch(t *testing.T) {
0, 4, // children bitmap
// missing children scale encoded data
}),
nodeVariant: branchVariant,
nodeVariant: BranchVariant,
partialKeyLength: 1,
errWrapped: ErrDecodeChildHash,
errMessage: "cannot decode child hash: at index 10: reading byte: EOF",
Expand All @@ -221,7 +221,7 @@ func Test_decodeBranch(t *testing.T) {
scaleEncodedChildHash,
}),
),
nodeVariant: branchVariant,
nodeVariant: BranchVariant,
partialKeyLength: 1,
branch: &Node{
PartialKey: []byte{9},
Expand All @@ -243,7 +243,7 @@ func Test_decodeBranch(t *testing.T) {
// missing encoded branch storage value
}),
),
nodeVariant: branchWithValueVariant,
nodeVariant: BranchWithValueVariant,
partialKeyLength: 1,
errWrapped: ErrDecodeStorageValue,
errMessage: "cannot decode storage value: reading byte: EOF",
Expand All @@ -255,7 +255,7 @@ func Test_decodeBranch(t *testing.T) {
scaleEncodeBytes(t, 7, 8, 9), // branch storage value
scaleEncodedChildHash,
})),
nodeVariant: branchWithValueVariant,
nodeVariant: BranchWithValueVariant,
partialKeyLength: 1,
branch: &Node{
PartialKey: []byte{9},
Expand All @@ -277,7 +277,7 @@ func Test_decodeBranch(t *testing.T) {
scaleEncodeBytes(t, 1), // branch storage value
{0}, // garbage inlined node
})),
nodeVariant: branchWithValueVariant,
nodeVariant: BranchWithValueVariant,
partialKeyLength: 1,
errWrapped: io.EOF,
errMessage: "decoding inlined child at index 0: " +
Expand All @@ -289,25 +289,25 @@ func Test_decodeBranch(t *testing.T) {
{0b0000_0011, 0b0000_0000}, // children bitmap
// top level inlined leaf less than 32 bytes
scaleEncodeByteSlice(t, concatByteSlices([][]byte{
{leafVariant.bits | 1}, // partial key length of 1
{LeafVariant.bits | 1}, // partial key length of 1
{2}, // key data
scaleEncodeBytes(t, 2), // storage value data
})),
// top level inlined branch less than 32 bytes
scaleEncodeByteSlice(t, concatByteSlices([][]byte{
{branchWithValueVariant.bits | 1}, // partial key length of 1
{BranchWithValueVariant.bits | 1}, // partial key length of 1
{3}, // key data
{0b0000_0001, 0b0000_0000}, // children bitmap
scaleEncodeBytes(t, 3), // branch storage value
// bottom level leaf
scaleEncodeByteSlice(t, concatByteSlices([][]byte{
{leafVariant.bits | 1}, // partial key length of 1
{LeafVariant.bits | 1}, // partial key length of 1
{4}, // key data
scaleEncodeBytes(t, 4), // storage value data
})),
})),
})),
nodeVariant: branchVariant,
nodeVariant: BranchVariant,
partialKeyLength: 1,
branch: &Node{
PartialKey: []byte{1},
Expand Down Expand Up @@ -349,7 +349,7 @@ func Test_decodeLeaf(t *testing.T) {

testCases := map[string]struct {
reader io.Reader
variant variant
variant Variant
partialKeyLength uint16
leaf *Node
errWrapped error
Expand All @@ -359,7 +359,7 @@ func Test_decodeLeaf(t *testing.T) {
reader: bytes.NewBuffer([]byte{
// missing key data byte
}),
variant: leafVariant,
variant: LeafVariant,
partialKeyLength: 1,
errWrapped: io.EOF,
errMessage: "cannot decode key: reading from reader: EOF",
Expand All @@ -369,7 +369,7 @@ func Test_decodeLeaf(t *testing.T) {
{9}, // key data
{255, 255}, // bad storage value data
})),
variant: leafVariant,
variant: LeafVariant,
partialKeyLength: 1,
errWrapped: ErrDecodeStorageValue,
errMessage: "cannot decode storage value: unknown prefix for compact uint: 255",
Expand All @@ -379,7 +379,7 @@ func Test_decodeLeaf(t *testing.T) {
9, // key data
// missing storage value data
}),
variant: leafVariant,
variant: LeafVariant,
partialKeyLength: 1,
errWrapped: ErrDecodeStorageValue,
errMessage: "cannot decode storage value: reading byte: EOF",
Expand All @@ -389,7 +389,7 @@ func Test_decodeLeaf(t *testing.T) {
{9}, // key data
scaleEncodeByteSlice(t, []byte{}), // results to []byte{0}
})),
variant: leafVariant,
variant: LeafVariant,
partialKeyLength: 1,
leaf: &Node{
PartialKey: []byte{9},
Expand All @@ -401,7 +401,7 @@ func Test_decodeLeaf(t *testing.T) {
{9}, // key data
scaleEncodeBytes(t, 1, 2, 3, 4, 5), // storage value data
})),
variant: leafVariant,
variant: LeafVariant,
partialKeyLength: 1,
leaf: &Node{
PartialKey: []byte{9},
Expand Down
2 changes: 1 addition & 1 deletion internal/trie/node/encode_decode_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func Test_Branch_Encode_Decode(t *testing.T) {
err := testCase.branchToEncode.Encode(buffer)
require.NoError(t, err)

nodeVariant, partialKeyLength, err := decodeHeader(buffer)
nodeVariant, partialKeyLength, err := DecodeHeader(buffer)
require.NoError(t, err)

resultBranch, err := decodeBranch(buffer, nodeVariant, partialKeyLength)
Expand Down
Loading