Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Paritytech/trie proofs are incompatible with gossamer/lib/trie #2329

Closed
seunlanlege opened this issue Feb 24, 2022 · 8 comments
Closed

Paritytech/trie proofs are incompatible with gossamer/lib/trie #2329

seunlanlege opened this issue Feb 24, 2022 · 8 comments

Comments

@seunlanlege
Copy link

Describe the bug

Read proofs generated by the rust implementation of the substrate node don't seem to work with the lib/trie library.

Screenshot 2022-02-24 at 11 33 20 AM

Screenshot 2022-02-24 at 11 35 54 AM

func TestTrieProof(t *testing.T) {
	key, err := hex.DecodeString("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb")
	if err != nil {
		panic(err)
	}
	root, err := hex.DecodeString("dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0")
	if err != nil {
		panic(err)
	}
	bytes1, err := hex.DecodeString("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c")
	if err != nil {
		panic(err)
	}
	bytes2, err := hex.DecodeString("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000")
	if err != nil {
		panic(err)
	}
	bytes3, err := hex.DecodeString("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc")
	if err != nil {
		panic(err)
	}
	var proof = [][]byte{
		bytes1, bytes2, bytes3,
	}
	trie := trie.NewEmptyTrie()
	errd := trie.LoadFromProof(proof, root)
	if errd != nil {
		panic(err)
	}
	value :=  trie.Get(key) // value here is empty
	fmt.Printf("value: %s\n", value)
}

Expected Behavior

    #[test]
    fn test_state_proof_verification() {
        let key = hex!("f0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb").to_vec();
        let state_root = H256::from(hex!(
            "dc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0"
        ));

        let proof = vec![
            hex!("80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c").to_vec(),
            hex!("9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000").to_vec(),
            hex!("8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc").to_vec(),
        ];
        let db = StorageProof::new(proof).into_memory_db();

        let trie =
            sp_trie::TrieDB::<sp_trie::LayoutV0<BlakeTwo256>>::new(&db, &state_root).unwrap();

        let value = trie.get(&key).unwrap().unwrap(); // actually gets the value from the trie proof

        let timestamp: u64 = codec::Decode::decode(&mut &value[..]).unwrap();

        println!("timestamp from proof: {}", timestamp);
    }
  • go version: 1.17.6
  • gossamer version: v0.6.1
  • gossamer commit tag: v0.6.1
  • gossamer commit hash: e1f7f96
  • operating system: mac 12.1
@notbdu
Copy link
Contributor

notbdu commented Mar 3, 2022

Adding some additional notes here.

// Decodes 3 nodes from proof (one root, two branch/leaf nodes)
decoded node -> [] branch key=0x childrenBitmap=1111110111111111 value=0x dirty=false
found root in proof
0x76c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d6 branch key=0x0c0306050c030c0f05090d0607010e0b07020d0a000e070a040101030c04 childrenBitmap=1000010000 value=0x dirty=false
decoded node -> [12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4] branch key=0x0c0306050c030c0f05090d0607010e0b07020d0a000e070a040101030c04 childrenBitmap=1000010000 value=0x dirty=false
0x3360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c branch key=0x childrenBitmap=100000000101 value=0x dirty=false
decoded node -> [] branch key=0x childrenBitmap=100000000101 value=0x dirty=false

// Assign child to root @ idx 15
ASSIGN NODE to parent -> branch key=0x childrenBitmap=1111110111111111 value=0x dirty=false idx -> 15 child key -> [] child -> branch key=0x childrenBitmap=100000000101 value=0x dirty=false
// Assign child to branch @ idx 0
ASSIGN NODE to parent -> branch key=0x childrenBitmap=100000000101 value=0x dirty=false idx -> 0 child key -> [12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4] child -> branch key=0x0c0306050c030c0f05090d0607010e0b07020d0a000e070a040101030c04 childrenBitmap=1000010000 value=0x dirty=false

look for key -> [15 0 12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4 9 15 1 15 0 5 1 5 15 4 6 2 12 13 12 15 8 4 14 0 15 1 13 6 0 4 5 13 15 12 11 11]

So the key of the leaf node in the example given should be:
[15 0 12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4]

If we look side by side we get:

[15 0 12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4 9 15 1 15 0 5 1 5 15 4 6 2 12 13 12 15 8 4 14 0 15 1 13 6 0 4 5 13 15 12 11 11]
[15 0 12 3 6 5 12 3 12 15 5 9 13 6 7 1 14 11 7 2 13 10 0 14 7 10 4 1 1 3 12 4]

It's almost a match but as we can see above the key we're attempting to search for is much longer.

@seunlanlege
Copy link
Author

So i decoded all the nodes in rust and here's their structure:

// 0x8005088076c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d680c1638f702aaa71e4b78cc8538ecae03e827bb494cc54279606b201ec071a5e24806d2a1e6d5236e1e13c5a5c84831f5f5383f97eba32df6f9faf80e32cf2f129bc
NibbledBranch(
    ,
    [
        Some(Hash([118, 198, 110, 40, 113, 180, 254, 3, 125, 17, 46, 191, 251, 59, 252, 139, 216, 58, 78, 194, 96, 71, 245, 142, 226, 223, 123, 228, 233, 235, 227, 214])),
        None,
        Some(Hash([193, 99, 143, 112, 42, 170, 113, 228, 183, 140, 200, 83, 142, 202, 224, 62, 130, 123, 180, 148, 204, 84, 39, 150, 6, 178, 1, 236, 7, 26, 94, 36])),
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        Some(Hash([109, 42, 30, 109, 82, 54, 225, 225, 60, 90, 92, 132, 131, 31, 95, 83, 131, 249, 126, 186, 50, 223, 111, 159, 175, 128, 227, 44, 242, 241, 41, 188])),
        None,
        None,
        None,
        None
    ],
    None
)

// 0x9ec365c3cf59d671eb72da0e7a4113c41002505f0e7b9012096b41c4eb3aaf947f6ea429080000685f0f1f0515f462cdcf84e0f1d6045dfcbb20865c4a2b7f010000
NibbledBranch(
    c'3'6'5'c'3'c'f'5'9'd'6'7'1'e'b'7'2'd'a'0'e'7'a'4'1'1'3'c'4,
    [
        None,
        None,
        None,
        None,
        Some(Inline([95, 14, 123, 144, 18, 9, 107, 65, 196, 235, 58, 175, 148, 127, 110, 164, 41, 8, 0, 0])),
        None,
        None,
        None,
        None,
        Some(Inline([95, 15, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187, 32, 134, 92, 74, 43, 127, 1, 0, 0])),
        None,
        None,
        None,
        None,
        None,
        None
    ],
    None
)

// 0x80fffd8028b54b9a0a90d41b7941c43e6a0597d5914e3b62bdcb244851b9fc806c28ea2480d5ba6d50586692888b0c2f5b3c3fc345eb3a2405996f025ed37982ca396f5ed580bd281c12f20f06077bffd56b2f8b6431ee6c9fd11fed9c22db86cea849aeff2280afa1e1b5ce72ea1675e5e69be85e98fbfb660691a76fee9229f758a75315f2bc80aafc60caa3519d4b861e6b8da226266a15060e2071bba4184e194da61dfb208e809d3f6ae8f655009551de95ae1ef863f6771522fd5c0475a50ff53c5c8169b5888024a760a8f6c27928ae9e2fed9968bc5f6e17c3ae647398d8a615e5b2bb4b425f8085a0da830399f25fca4b653de654ffd3c92be39f3ae4f54e7c504961b5bd00cf80c2d44d371e5fc1f50227d7491ad65ad049630361cefb4ab1844831237609f08380c644938921d14ae611f3a90991af8b7f5bdb8fa361ee2c646c849bca90f491e6806e729ad43a591cd1321762582782bbe4ed193c6f583ec76013126f7f786e376280509bb016f2887d12137e73d26d7ddcd7f9c8ff458147cb9d309494655fe68de180009f8697d760fbe020564b07f407e6aad58ba9451b3d2d88b3ee03e12db7c47480952dcc0804e1120508a1753f1de4aa5b7481026a3320df8b48e918f0cecbaed3803360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c
NibbledBranch(
    ,
    [
        Some(Hash([40, 181, 75, 154, 10, 144, 212, 27, 121, 65, 196, 62, 106, 5, 151, 213, 145, 78, 59, 98, 189, 203, 36, 72, 81, 185, 252, 128, 108, 40, 234, 36])),
        Some(Hash([213, 186, 109, 80, 88, 102, 146, 136, 139, 12, 47, 91, 60, 63, 195, 69, 235, 58, 36, 5, 153, 111, 2, 94, 211, 121, 130, 202, 57, 111, 94, 213])),
        Some(Hash([189, 40, 28, 18, 242, 15, 6, 7, 123, 255, 213, 107, 47, 139, 100, 49, 238, 108, 159, 209, 31, 237, 156, 34, 219, 134, 206, 168, 73, 174, 255, 34])),
        Some(Hash([175, 161, 225, 181, 206, 114, 234, 22, 117, 229, 230, 155, 232, 94, 152, 251, 251, 102, 6, 145, 167, 111, 238, 146, 41, 247, 88, 167, 83, 21, 242, 188])),
        Some(Hash([170, 252, 96, 202, 163, 81, 157, 75, 134, 30, 107, 141, 162, 38, 38, 106, 21, 6, 14, 32, 113, 187, 164, 24, 78, 25, 77, 166, 29, 251, 32, 142])),
        Some(Hash([157, 63, 106, 232, 246, 85, 0, 149, 81, 222, 149, 174, 30, 248, 99, 246, 119, 21, 34, 253, 92, 4, 117, 165, 15, 245, 60, 92, 129, 105, 181, 136])),
        Some(Hash([36, 167, 96, 168, 246, 194, 121, 40, 174, 158, 47, 237, 153, 104, 188, 95, 110, 23, 195, 174, 100, 115, 152, 216, 166, 21, 229, 178, 187, 75, 66, 95])),
        Some(Hash([133, 160, 218, 131, 3, 153, 242, 95, 202, 75, 101, 61, 230, 84, 255, 211, 201, 43, 227, 159, 58, 228, 245, 78, 124, 80, 73, 97, 181, 189, 0, 207])),
        Some(Hash([194, 212, 77, 55, 30, 95, 193, 245, 2, 39, 215, 73, 26, 214, 90, 208, 73, 99, 3, 97, 206, 251, 74, 177, 132, 72, 49, 35, 118, 9, 240, 131])),
        None,
        Some(Hash([198, 68, 147, 137, 33, 209, 74, 230, 17, 243, 169, 9, 145, 175, 139, 127, 91, 219, 143, 163, 97, 238, 44, 100, 108, 132, 155, 202, 144, 244, 145, 230])),
        Some(Hash([110, 114, 154, 212, 58, 89, 28, 209, 50, 23, 98, 88, 39, 130, 187, 228, 237, 25, 60, 111, 88, 62, 199, 96, 19, 18, 111, 127, 120, 110, 55, 98])),
        Some(Hash([80, 155, 176, 22, 242, 136, 125, 18, 19, 126, 115, 210, 109, 125, 220, 215, 249, 200, 255, 69, 129, 71, 203, 157, 48, 148, 148, 101, 95, 230, 141, 225])),
        Some(Hash([0, 159, 134, 151, 215, 96, 251, 224, 32, 86, 75, 7, 244, 7, 230, 170, 213, 139, 169, 69, 27, 61, 45, 136, 179, 238, 3, 225, 45, 183, 196, 116])),
        Some(Hash([149, 45, 204, 8, 4, 225, 18, 5, 8, 161, 117, 63, 29, 228, 170, 91, 116, 129, 2, 106, 51, 32, 223, 139, 72, 233, 24, 240, 206, 203, 174, 211])),
        Some(Hash([51, 96, 191, 148, 143, 221, 196, 3, 211, 69, 6, 64, 130, 232, 57, 61, 122, 26, 173, 122, 25, 8, 31, 109, 2, 217, 67, 88, 242, 66, 184, 108]))
    ],
    None
)

@notbdu
Copy link
Contributor

notbdu commented Mar 8, 2022

Hmm, looks like there's a leaf node in the parity/trie impl that's not present in the go version?

Pasting logs from a lookup operation in partiy/trie.

// insert three items from proof into memory db
inserting item from proof -> [128, 255, 253, 128, 40, 181, 75, 154, 10, 144, 212, 27, 121, 65, 196, 62, 106, 5, 151, 213, 145, 78, 59, 98, 189, 203, 36, 72, 81, 185, 252, 128, 108, 40, 234, 36, 128, 213, 186, 109, 80, 88, 102, 146, 136, 139, 12, 47, 91, 60, 63, 195, 69, 235, 58, 36, 5, 153, 111, 2, 94, 211, 121, 130, 202, 57, 111, 94, 213, 128, 189, 40, 28, 18, 242, 15, 6, 7, 123, 255, 213, 107, 47, 139, 100, 49, 238, 108, 159, 209, 31, 237, 156, 34, 219, 134, 206, 168, 73, 174, 255, 34, 128, 175, 161, 225, 181, 206, 114, 234, 22, 117, 229, 230, 155, 232, 94, 152, 251, 251, 102, 6, 145, 167, 111, 238, 146, 41, 247, 88, 167, 83, 21, 242, 188, 128, 170, 252, 96, 202, 163, 81, 157, 75, 134, 30, 107, 141, 162, 38, 38, 106, 21, 6, 14, 32, 113, 187, 164, 24, 78, 25, 77, 166, 29, 251, 32, 142, 128, 157, 63, 106, 232, 246, 85, 0, 149, 81, 222, 149, 174, 30, 248, 99, 246, 119, 21, 34, 253, 92, 4, 117, 165, 15, 245, 60, 92, 129, 105, 181, 136, 128, 36, 167, 96, 168, 246, 194, 121, 40, 174, 158, 47, 237, 153, 104, 188, 95, 110, 23, 195, 174, 100, 115, 152, 216, 166, 21, 229, 178, 187, 75, 66, 95, 128, 133, 160, 218, 131, 3, 153, 242, 95, 202, 75, 101, 61, 230, 84, 255, 211, 201, 43, 227, 159, 58, 228, 245, 78, 124, 80, 73, 97, 181, 189, 0, 207, 128, 194, 212, 77, 55, 30, 95, 193, 245, 2, 39, 215, 73, 26, 214, 90, 208, 73, 99, 3, 97, 206, 251, 74, 177, 132, 72, 49, 35, 118, 9, 240, 131, 128, 198, 68, 147, 137, 33, 209, 74, 230, 17, 243, 169, 9, 145, 175, 139, 127, 91, 219, 143, 163, 97, 238, 44, 100, 108, 132, 155, 202, 144, 244, 145, 230, 128, 110, 114, 154, 212, 58, 89, 28, 209, 50, 23, 98, 88, 39, 130, 187, 228, 237, 25, 60, 111, 88, 62, 199, 96, 19, 18, 111, 127, 120, 110, 55, 98, 128, 80, 155, 176, 22, 242, 136, 125, 18, 19, 126, 115, 210, 109, 125, 220, 215, 249, 200, 255, 69, 129, 71, 203, 157, 48, 148, 148, 101, 95, 230, 141, 225, 128, 0, 159, 134, 151, 215, 96, 251, 224, 32, 86, 75, 7, 244, 7, 230, 170, 213, 139, 169, 69, 27, 61, 45, 136, 179, 238, 3, 225, 45, 183, 196, 116, 128, 149, 45, 204, 8, 4, 225, 18, 5, 8, 161, 117, 63, 29, 228, 170, 91, 116, 129, 2, 106, 51, 32, 223, 139, 72, 233, 24, 240, 206, 203, 174, 211, 128, 51, 96, 191, 148, 143, 221, 196, 3, 211, 69, 6, 64, 130, 232, 57, 61, 122, 26, 173, 122, 25, 8, 31, 109, 2, 217, 67, 88, 242, 66, 184, 108]
inserting item from proof -> [158, 195, 101, 195, 207, 89, 214, 113, 235, 114, 218, 14, 122, 65, 19, 196, 16, 2, 80, 95, 14, 123, 144, 18, 9, 107, 65, 196, 235, 58, 175, 148, 127, 110, 164, 41, 8, 0, 0, 104, 95, 15, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187, 32, 134, 92, 74, 43, 127, 1, 0, 0]
inserting item from proof -> [128, 5, 8, 128, 118, 198, 110, 40, 113, 180, 254, 3, 125, 17, 46, 191, 251, 59, 252, 139, 216, 58, 78, 194, 96, 71, 245, 142, 226, 223, 123, 228, 233, 235, 227, 214, 128, 193, 99, 143, 112, 42, 170, 113, 228, 183, 140, 200, 83, 142, 202, 224, 62, 130, 123, 180, 148, 204, 84, 39, 150, 6, 178, 1, 236, 7, 26, 94, 36, 128, 109, 42, 30, 109, 82, 54, 225, 225, 60, 90, 92, 132, 131, 31, 95, 83, 131, 249, 126, 186, 50, 223, 111, 159, 175, 128, 227, 44, 242, 241, 41, 188]

get key nibbles: f'0'c'3'6'5'c'3'c'f'5'9'd'6'7'1'e'b'7'2'd'a'0'e'7'a'4'1'1'3'c'4'9'f'1'f'0'5'1'5'f'4'6'2'c'd'c'f'8'4'e'0'f'1'd'6'0'4'5'd'f'c'b'b

// decode four nodes on lookup
decoded -> NibbledBranch(, [Some(Hash([40, 181, 75, 154, 10, 144, 212, 27, 121, 65, 196, 62, 106, 5, 151, 213, 145, 78, 59, 98, 189, 203, 36, 72, 81, 185, 252, 128, 108, 40, 234, 36])), Some(Hash([213, 186, 109, 80, 88, 102, 146, 136, 139, 12, 47, 91, 60, 63, 195, 69, 235, 58, 36, 5, 153, 111, 2, 94, 211, 121, 130, 202, 57, 111, 94, 213])), Some(Hash([189, 40, 28, 18, 242, 15, 6, 7, 123, 255, 213, 107, 47, 139, 100, 49, 238, 108, 159, 209, 31, 237, 156, 34, 219, 134, 206, 168, 73, 174, 255, 34])), Some(Hash([175, 161, 225, 181, 206, 114, 234, 22, 117, 229, 230, 155, 232, 94, 152, 251, 251, 102, 6, 145, 167, 111, 238, 146, 41, 247, 88, 167, 83, 21, 242, 188])), Some(Hash([170, 252, 96, 202, 163, 81, 157, 75, 134, 30, 107, 141, 162, 38, 38, 106, 21, 6, 14, 32, 113, 187, 164, 24, 78, 25, 77, 166, 29, 251, 32, 142])), Some(Hash([157, 63, 106, 232, 246, 85, 0, 149, 81, 222, 149, 174, 30, 248, 99, 246, 119, 21, 34, 253, 92, 4, 117, 165, 15, 245, 60, 92, 129, 105, 181, 136])), Some(Hash([36, 167, 96, 168, 246, 194, 121, 40, 174, 158, 47, 237, 153, 104, 188, 95, 110, 23, 195, 174, 100, 115, 152, 216, 166, 21, 229, 178, 187, 75, 66, 95])), Some(Hash([133, 160, 218, 131, 3, 153, 242, 95, 202, 75, 101, 61, 230, 84, 255, 211, 201, 43, 227, 159, 58, 228, 245, 78, 124, 80, 73, 97, 181, 189, 0, 207])), Some(Hash([194, 212, 77, 55, 30, 95, 193, 245, 2, 39, 215, 73, 26, 214, 90, 208, 73, 99, 3, 97, 206, 251, 74, 177, 132, 72, 49, 35, 118, 9, 240, 131])), None, Some(Hash([198, 68, 147, 137, 33, 209, 74, 230, 17, 243, 169, 9, 145, 175, 139, 127, 91, 219, 143, 163, 97, 238, 44, 100, 108, 132, 155, 202, 144, 244, 145, 230])), Some(Hash([110, 114, 154, 212, 58, 89, 28, 209, 50, 23, 98, 88, 39, 130, 187, 228, 237, 25, 60, 111, 88, 62, 199, 96, 19, 18, 111, 127, 120, 110, 55, 98])), Some(Hash([80, 155, 176, 22, 242, 136, 125, 18, 19, 126, 115, 210, 109, 125, 220, 215, 249, 200, 255, 69, 129, 71, 203, 157, 48, 148, 148, 101, 95, 230, 141, 225])), Some(Hash([0, 159, 134, 151, 215, 96, 251, 224, 32, 86, 75, 7, 244, 7, 230, 170, 213, 139, 169, 69, 27, 61, 45, 136, 179, 238, 3, 225, 45, 183, 196, 116])), Some(Hash([149, 45, 204, 8, 4, 225, 18, 5, 8, 161, 117, 63, 29, 228, 170, 91, 116, 129, 2, 106, 51, 32, 223, 139, 72, 233, 24, 240, 206, 203, 174, 211])), Some(Hash([51, 96, 191, 148, 143, 221, 196, 3, 211, 69, 6, 64, 130, 232, 57, 61, 122, 26, 173, 122, 25, 8, 31, 109, 2, 217, 67, 88, 242, 66, 184, 108]))], None)
decoded -> NibbledBranch(, [Some(Hash([118, 198, 110, 40, 113, 180, 254, 3, 125, 17, 46, 191, 251, 59, 252, 139, 216, 58, 78, 194, 96, 71, 245, 142, 226, 223, 123, 228, 233, 235, 227, 214])), None, Some(Hash([193, 99, 143, 112, 42, 170, 113, 228, 183, 140, 200, 83, 142, 202, 224, 62, 130, 123, 180, 148, 204, 84, 39, 150, 6, 178, 1, 236, 7, 26, 94, 36])), None, None, None, None, None, None, None, None, Some(Hash([109, 42, 30, 109, 82, 54, 225, 225, 60, 90, 92, 132, 131, 31, 95, 83, 131, 249, 126, 186, 50, 223, 111, 159, 175, 128, 227, 44, 242, 241, 41, 188])), None, None, None, None], None)
decoded -> NibbledBranch(c'3'6'5'c'3'c'f'5'9'd'6'7'1'e'b'7'2'd'a'0'e'7'a'4'1'1'3'c'4, [None, None, None, None, Some(Inline([95, 14, 123, 144, 18, 9, 107, 65, 196, 235, 58, 175, 148, 127, 110, 164, 41, 8, 0, 0])), None, None, None, None, Some(Inline([95, 15, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187, 32, 134, 92, 74, 43, 127, 1, 0, 0])), None, None, None, None, None, None], None)
decoded -> Leaf(f'1'f'0'5'1'5'f'4'6'2'c'd'c'f'8'4'e'0'f'1'd'6'0'4'5'd'f'c'b'b, Inline([134, 92, 74, 43, 127, 1, 0, 0]))

Looks like the key is a perfect match above, yielding a found value.

We're directly calling db.get(); on lookup and the db is bootstrapped only from the proof data. So somehow the 3 proof items are converted into 4 nodes on db insert?

@notbdu
Copy link
Contributor

notbdu commented Mar 8, 2022

Ah, makes more sense now after additional logs. The leaf data is inlined haha 🤦‍♂️.

We lookup the last branch node and decode node data for that node and then the inlined leaf node data on the next iteration.

inserting item from proof -> [128, 255, 253, 128, 40, 181, 75, 154, 10, 144, 212, 27, 121, 65, 196, 62, 106, 5, 151, 213, 145, 78, 59, 98, 189, 203, 36, 72, 81, 185, 252, 128, 108, 40, 234, 36, 128, 213, 186, 109, 80, 88, 102, 146, 136, 139, 12, 47, 91, 60, 63, 195, 69, 235, 58, 36, 5, 153, 111, 2, 94, 211, 121, 130, 202, 57, 111, 94, 213, 128, 189, 40, 28, 18, 242, 15, 6, 7, 123, 255, 213, 107, 47, 139, 100, 49, 238, 108, 159, 209, 31, 237, 156, 34, 219, 134, 206, 168, 73, 174, 255, 34, 128, 175, 161, 225, 181, 206, 114, 234, 22, 117, 229, 230, 155, 232, 94, 152, 251, 251, 102, 6, 145, 167, 111, 238, 146, 41, 247, 88, 167, 83, 21, 242, 188, 128, 170, 252, 96, 202, 163, 81, 157, 75, 134, 30, 107, 141, 162, 38, 38, 106, 21, 6, 14, 32, 113, 187, 164, 24, 78, 25, 77, 166, 29, 251, 32, 142, 128, 157, 63, 106, 232, 246, 85, 0, 149, 81, 222, 149, 174, 30, 248, 99, 246, 119, 21, 34, 253, 92, 4, 117, 165, 15, 245, 60, 92, 129, 105, 181, 136, 128, 36, 167, 96, 168, 246, 194, 121, 40, 174, 158, 47, 237, 153, 104, 188, 95, 110, 23, 195, 174, 100, 115, 152, 216, 166, 21, 229, 178, 187, 75, 66, 95, 128, 133, 160, 218, 131, 3, 153, 242, 95, 202, 75, 101, 61, 230, 84, 255, 211, 201, 43, 227, 159, 58, 228, 245, 78, 124, 80, 73, 97, 181, 189, 0, 207, 128, 194, 212, 77, 55, 30, 95, 193, 245, 2, 39, 215, 73, 26, 214, 90, 208, 73, 99, 3, 97, 206, 251, 74, 177, 132, 72, 49, 35, 118, 9, 240, 131, 128, 198, 68, 147, 137, 33, 209, 74, 230, 17, 243, 169, 9, 145, 175, 139, 127, 91, 219, 143, 163, 97, 238, 44, 100, 108, 132, 155, 202, 144, 244, 145, 230, 128, 110, 114, 154, 212, 58, 89, 28, 209, 50, 23, 98, 88, 39, 130, 187, 228, 237, 25, 60, 111, 88, 62, 199, 96, 19, 18, 111, 127, 120, 110, 55, 98, 128, 80, 155, 176, 22, 242, 136, 125, 18, 19, 126, 115, 210, 109, 125, 220, 215, 249, 200, 255, 69, 129, 71, 203, 157, 48, 148, 148, 101, 95, 230, 141, 225, 128, 0, 159, 134, 151, 215, 96, 251, 224, 32, 86, 75, 7, 244, 7, 230, 170, 213, 139, 169, 69, 27, 61, 45, 136, 179, 238, 3, 225, 45, 183, 196, 116, 128, 149, 45, 204, 8, 4, 225, 18, 5, 8, 161, 117, 63, 29, 228, 170, 91, 116, 129, 2, 106, 51, 32, 223, 139, 72, 233, 24, 240, 206, 203, 174, 211, 128, 51, 96, 191, 148, 143, 221, 196, 3, 211, 69, 6, 64, 130, 232, 57, 61, 122, 26, 173, 122, 25, 8, 31, 109, 2, 217, 67, 88, 242, 66, 184, 108]
insert key -> 0xdc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0
inserting item from proof -> [158, 195, 101, 195, 207, 89, 214, 113, 235, 114, 218, 14, 122, 65, 19, 196, 16, 2, 80, 95, 14, 123, 144, 18, 9, 107, 65, 196, 235, 58, 175, 148, 127, 110, 164, 41, 8, 0, 0, 104, 95, 15, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187, 32, 134, 92, 74, 43, 127, 1, 0, 0]
insert key -> 0x76c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d6
inserting item from proof -> [128, 5, 8, 128, 118, 198, 110, 40, 113, 180, 254, 3, 125, 17, 46, 191, 251, 59, 252, 139, 216, 58, 78, 194, 96, 71, 245, 142, 226, 223, 123, 228, 233, 235, 227, 214, 128, 193, 99, 143, 112, 42, 170, 113, 228, 183, 140, 200, 83, 142, 202, 224, 62, 130, 123, 180, 148, 204, 84, 39, 150, 6, 178, 1, 236, 7, 26, 94, 36, 128, 109, 42, 30, 109, 82, 54, 225, 225, 60, 90, 92, 132, 131, 31, 95, 83, 131, 249, 126, 186, 50, 223, 111, 159, 175, 128, 227, 44, 242, 241, 41, 188]
insert key -> 0x3360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c
get key: [240, 195, 101, 195, 207, 89, 214, 113, 235, 114, 218, 14, 122, 65, 19, 196, 159, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187]

get key nibbles: f'0'c'3'6'5'c'3'c'f'5'9'd'6'7'1'e'b'7'2'd'a'0'e'7'a'4'1'1'3'c'4'9'f'1'f'0'5'1'5'f'4'6'2'c'd'c'f'8'4'e'0'f'1'd'6'0'4'5'd'f'c'b'b

lookup get hash -> 0xdc4887669c2a6b3462e9557aa3105a66a02b6ec3b21784613de78c95dc3cbbe0, key -> ([], None)
decoded -> NibbledBranch(, [Some(Hash([40, 181, 75, 154, 10, 144, 212, 27, 121, 65, 196, 62, 106, 5, 151, 213, 145, 78, 59, 98, 189, 203, 36, 72, 81, 185, 252, 128, 108, 40, 234, 36])), Some(Hash([213, 186, 109, 80, 88, 102, 146, 136, 139, 12, 47, 91, 60, 63, 195, 69, 235, 58, 36, 5, 153, 111, 2, 94, 211, 121, 130, 202, 57, 111, 94, 213])), Some(Hash([189, 40, 28, 18, 242, 15, 6, 7, 123, 255, 213, 107, 47, 139, 100, 49, 238, 108, 159, 209, 31, 237, 156, 34, 219, 134, 206, 168, 73, 174, 255, 34])), Some(Hash([175, 161, 225, 181, 206, 114, 234, 22, 117, 229, 230, 155, 232, 94, 152, 251, 251, 102, 6, 145, 167, 111, 238, 146, 41, 247, 88, 167, 83, 21, 242, 188])), Some(Hash([170, 252, 96, 202, 163, 81, 157, 75, 134, 30, 107, 141, 162, 38, 38, 106, 21, 6, 14, 32, 113, 187, 164, 24, 78, 25, 77, 166, 29, 251, 32, 142])), Some(Hash([157, 63, 106, 232, 246, 85, 0, 149, 81, 222, 149, 174, 30, 248, 99, 246, 119, 21, 34, 253, 92, 4, 117, 165, 15, 245, 60, 92, 129, 105, 181, 136])), Some(Hash([36, 167, 96, 168, 246, 194, 121, 40, 174, 158, 47, 237, 153, 104, 188, 95, 110, 23, 195, 174, 100, 115, 152, 216, 166, 21, 229, 178, 187, 75, 66, 95])), Some(Hash([133, 160, 218, 131, 3, 153, 242, 95, 202, 75, 101, 61, 230, 84, 255, 211, 201, 43, 227, 159, 58, 228, 245, 78, 124, 80, 73, 97, 181, 189, 0, 207])), Some(Hash([194, 212, 77, 55, 30, 95, 193, 245, 2, 39, 215, 73, 26, 214, 90, 208, 73, 99, 3, 97, 206, 251, 74, 177, 132, 72, 49, 35, 118, 9, 240, 131])), None, Some(Hash([198, 68, 147, 137, 33, 209, 74, 230, 17, 243, 169, 9, 145, 175, 139, 127, 91, 219, 143, 163, 97, 238, 44, 100, 108, 132, 155, 202, 144, 244, 145, 230])), Some(Hash([110, 114, 154, 212, 58, 89, 28, 209, 50, 23, 98, 88, 39, 130, 187, 228, 237, 25, 60, 111, 88, 62, 199, 96, 19, 18, 111, 127, 120, 110, 55, 98])), Some(Hash([80, 155, 176, 22, 242, 136, 125, 18, 19, 126, 115, 210, 109, 125, 220, 215, 249, 200, 255, 69, 129, 71, 203, 157, 48, 148, 148, 101, 95, 230, 141, 225])), Some(Hash([0, 159, 134, 151, 215, 96, 251, 224, 32, 86, 75, 7, 244, 7, 230, 170, 213, 139, 169, 69, 27, 61, 45, 136, 179, 238, 3, 225, 45, 183, 196, 116])), Some(Hash([149, 45, 204, 8, 4, 225, 18, 5, 8, 161, 117, 63, 29, 228, 170, 91, 116, 129, 2, 106, 51, 32, 223, 139, 72, 233, 24, 240, 206, 203, 174, 211])), Some(Hash([51, 96, 191, 148, 143, 221, 196, 3, 211, 69, 6, 64, 130, 232, 57, 61, 122, 26, 173, 122, 25, 8, 31, 109, 2, 217, 67, 88, 242, 66, 184, 108]))], None)
lookup get hash -> 0x3360bf948fddc403d345064082e8393d7a1aad7a19081f6d02d94358f242b86c, key -> ([], Some(240))
decoded -> NibbledBranch(, [Some(Hash([118, 198, 110, 40, 113, 180, 254, 3, 125, 17, 46, 191, 251, 59, 252, 139, 216, 58, 78, 194, 96, 71, 245, 142, 226, 223, 123, 228, 233, 235, 227, 214])), None, Some(Hash([193, 99, 143, 112, 42, 170, 113, 228, 183, 140, 200, 83, 142, 202, 224, 62, 130, 123, 180, 148, 204, 84, 39, 150, 6, 178, 1, 236, 7, 26, 94, 36])), None, None, None, None, None, None, None, None, Some(Hash([109, 42, 30, 109, 82, 54, 225, 225, 60, 90, 92, 132, 131, 31, 95, 83, 131, 249, 126, 186, 50, 223, 111, 159, 175, 128, 227, 44, 242, 241, 41, 188])), None, None, None, None], None)
lookup get hash -> 0x76c66e2871b4fe037d112ebffb3bfc8bd83a4ec26047f58ee2df7be4e9ebe3d6, key -> ([240], None)
decoded -> NibbledBranch(c'3'6'5'c'3'c'f'5'9'd'6'7'1'e'b'7'2'd'a'0'e'7'a'4'1'1'3'c'4, [None, None, None, None, Some(Inline([95, 14, 123, 144, 18, 9, 107, 65, 196, 235, 58, 175, 148, 127, 110, 164, 41, 8, 0, 0])), None, None, None, None, Some(Inline([95, 15, 31, 5, 21, 244, 98, 205, 207, 132, 224, 241, 214, 4, 93, 252, 187, 32, 134, 92, 74, 43, 127, 1, 0, 0])), None, None, None, None, None, None], None)
decoded -> Leaf(f'1'f'0'5'1'5'f'4'6'2'c'd'c'f'8'4'e'0'f'1'd'6'0'4'5'd'f'c'b'b, Inline([134, 92, 74, 43, 127, 1, 0, 0]))

@notbdu
Copy link
Contributor

notbdu commented Mar 8, 2022

Hmm I think the issue is that inlined leaf node key/value aren't being properly decoded from the proof payload?

var hash []byte
err := sd.Decode(&hash)
if err != nil {
return nil, fmt.Errorf("%w: at index %d: %s",
ErrDecodeChildHash, i, err)
}
branch.Children[i] = &Leaf{
HashDigest: hash,
}

It looks like it assumes that the leaf node is hashed and not inlined.

@notbdu
Copy link
Contributor

notbdu commented Mar 8, 2022

Pasting leaf node plan for decoding.

/// A `NodePlan` is a blueprint for decoding a node from a byte slice. The `NodePlan` is created
/// by parsing an encoded node and can be reused multiple times. This is useful as a `Node` borrows
/// from a byte slice and this struct does not.
///
/// The enum values mirror those of `Node` except that instead of byte slices, this struct stores
/// ranges that can be used to index into a large byte slice.
#[derive(Eq, PartialEq, Clone)]
#[cfg_attr(feature = "std", derive(Debug))]
pub enum NodePlan {
        /// Leaf node; has a partial key plan and value.
        Leaf { partial: NibbleSlicePlan, value: ValuePlan },
}

impl NodePlan {
        /// Build a node by decoding a byte slice according to the node plan. It is the responsibility
        /// of the caller to ensure that the node plan was created for the argument data, otherwise the
        /// call may decode incorrectly or panic.
        pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Node<'b> {
                match self {
                        NodePlan::Leaf { partial, value } => Node::Leaf(partial.build(data), value.build(data)),
                }
        }
}
 
 
 /// A `NibbleSlicePlan` is a blueprint for decoding a nibble slice from a byte slice. The
/// `NibbleSlicePlan` is created by parsing a byte slice and can be reused multiple times.
#[derive(Eq, PartialEq, Clone)]
#[cfg_attr(feature = "std", derive(Debug))]
pub struct NibbleSlicePlan {
        bytes: Range<usize>,
        offset: usize,
}

impl NibbleSlicePlan {
        /// Construct a nibble slice decode plan.
        pub fn new(bytes: Range<usize>, offset: usize) -> Self {
                NibbleSlicePlan { bytes, offset }
        }

        /// Returns the nibble length of the slice.
        pub fn len(&self) -> usize {
                (self.bytes.end - self.bytes.start) * nibble_ops::NIBBLE_PER_BYTE - self.offset
        }

        /// Build a nibble slice by decoding a byte slice according to the plan. It is the
        /// responsibility of the caller to ensure that the node plan was created for the argument
        /// data, otherwise the call may decode incorrectly or panic.
        pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> NibbleSlice<'b> {
                NibbleSlice::new_offset(&data[self.bytes.clone()], self.offset)
        }
}

/// Plan for value representation in `NodePlan`.
#[derive(Eq, PartialEq, Clone)]
#[cfg_attr(feature = "std", derive(Debug))]
pub enum ValuePlan {
        /// Range for byte representation in encoded node.
        Inline(Range<usize>),
        /// Range for hash in encoded node and original
        /// value size.
        Node(Range<usize>),
}

impl ValuePlan {
        /// Build a value slice by decoding a byte slice according to the plan.
        pub fn build<'a, 'b>(&'a self, data: &'b [u8]) -> Value<'b> {
                match self {
                        ValuePlan::Inline(range) => Value::Inline(&data[range.clone()]),
                        ValuePlan::Node(range) => Value::Node(&data[range.clone()], None),
                }
        }
}

thomasmodeneis added a commit to thomasmodeneis/gossamer that referenced this issue Apr 11, 2022
thomasmodeneis added a commit to thomasmodeneis/gossamer that referenced this issue Apr 13, 2022
thomasmodeneis added a commit to thomasmodeneis/gossamer that referenced this issue Apr 13, 2022
thomasmodeneis added a commit to thomasmodeneis/gossamer that referenced this issue Apr 14, 2022
@qdm12
Copy link
Contributor

qdm12 commented May 4, 2022

This is related to #2418 which I'm working on now. I'll update here soon.

@dimartiro
Copy link
Contributor

@seunlanlege @notbdu I've been checking this tests again and it's working now, probably related with my fix regarding the issue #3346

I'll close this issue

dimartiro added a commit that referenced this issue Aug 16, 2023
dimartiro added a commit that referenced this issue Aug 16, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

5 participants