Skip to content

Commit

Permalink
Rename Merkle value to node hash
Browse files Browse the repository at this point in the history
  • Loading branch information
@qdm12
qdm12 committed Dec 9, 2022
1 parent 54a9946 commit febbb27
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 51 deletions.
56 changes: 28 additions & 28 deletions internal/pruner/full.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import (
)

const (
lastPrunedKey = "last_pruned"
highestBlockNumberKey = "highest_block_number"
deletedMerkleValueKeyPrefix = "deleted_"
blockNumberToHashPrefix = "block_number_to_hash_"
lastPrunedKey = "last_pruned"
highestBlockNumberKey = "highest_block_number"
deletedNodeHashKeyPrefix = "deleted_"
blockNumberToHashPrefix = "block_number_to_hash_"
)

// FullNode prunes unneeded database keys for blocks older than the current
Expand Down Expand Up @@ -53,12 +53,12 @@ type journalKey struct {
}

type journalRecord struct {
// InsertedMerkleValues is the set of Merkle values of the trie nodes
// InsertedNodeHashes is the set of node hashes of the trie nodes
// inserted in the trie for the block.
InsertedMerkleValues map[string]struct{}
// DeletedMerkleValues is the set of Merkle values of the trie nodes
InsertedNodeHashes map[string]struct{}
// DeletedNodeHashes is the set of node hashes of the trie nodes
// removed from the trie for the block.
DeletedMerkleValues map[string]struct{}
DeletedNodeHashes map[string]struct{}
}

// NewFullNode creates a full node pruner.
Expand Down Expand Up @@ -106,7 +106,7 @@ func NewFullNode(journalDB JournalDatabase, storageDB ChainDBNewBatcher, retainB
// StoreJournalRecord stores the trie deltas impacting the storage database for a particular
// block hash. It prunes all block numbers falling off the window of block numbers to keep,
// before inserting the new record. It is thread safe to call.
func (p *FullNode) StoreJournalRecord(deletedMerkleValues, insertedMerkleValues map[string]struct{},
func (p *FullNode) StoreJournalRecord(deletedNodeHashes, insertedNodeHashes map[string]struct{},
blockHash common.Hash, blockNumber uint32) (err error) {
p.mutex.Lock()
defer p.mutex.Unlock()
Expand All @@ -121,7 +121,7 @@ func (p *FullNode) StoreJournalRecord(deletedMerkleValues, insertedMerkleValues
// WARNING: this must be before the pruning to avoid
// pruning still needed database keys.
journalDBBatch := p.journalDatabase.NewBatch()
err = p.handleInsertedKeys(insertedMerkleValues, blockNumber,
err = p.handleInsertedKeys(insertedNodeHashes, blockNumber,
blockHash, journalDBBatch)
if err != nil {
journalDBBatch.Reset()
Expand Down Expand Up @@ -167,8 +167,8 @@ func (p *FullNode) StoreJournalRecord(deletedMerkleValues, insertedMerkleValues
}

record := journalRecord{
InsertedMerkleValues: insertedMerkleValues,
DeletedMerkleValues: deletedMerkleValues,
InsertedNodeHashes: insertedNodeHashes,
DeletedNodeHashes: deletedNodeHashes,
}
err = storeJournalRecord(journalDBBatch, blockNumber, blockHash, record)
if err != nil {
Expand All @@ -185,29 +185,29 @@ func (p *FullNode) StoreJournalRecord(deletedMerkleValues, insertedMerkleValues
return nil
}

func (p *FullNode) handleInsertedKeys(insertedMerkleValues map[string]struct{},
func (p *FullNode) handleInsertedKeys(insertedNodeHashes map[string]struct{},
blockNumber uint32, blockHash common.Hash, journalDBBatch Putter) (err error) {
for insertedMerkleValue := range insertedMerkleValues {
err = p.handleInsertedKey(insertedMerkleValue, blockNumber, blockHash, journalDBBatch)
for insertedNodeHash := range insertedNodeHashes {
err = p.handleInsertedKey(insertedNodeHash, blockNumber, blockHash, journalDBBatch)
if err != nil {
return fmt.Errorf("handling inserted key 0x%x: %w",
[]byte(insertedMerkleValue), err)
[]byte(insertedNodeHash), err)
}
}

return nil
}

func (p *FullNode) handleInsertedKey(insertedMerkleValue string, blockNumber uint32,
func (p *FullNode) handleInsertedKey(insertedNodeHash string, blockNumber uint32,
blockHash common.Hash, journalDBBatch Putter) (err error) {
// Try to find if Merkle value was deleted in another block before
// Try to find if the node hash was deleted in another block before
// since we no longer want to prune it, as it was re-inserted.
journalKeyDeletedAt, err := p.journalDatabase.Get([]byte(deletedMerkleValueKeyPrefix + insertedMerkleValue))
merkleValueDeletedInAnotherBlock := errors.Is(err, chaindb.ErrKeyNotFound)
if !merkleValueDeletedInAnotherBlock {
journalKeyDeletedAt, err := p.journalDatabase.Get([]byte(deletedNodeHashKeyPrefix + insertedNodeHash))
nodeHashDeletedInAnotherBlock := errors.Is(err, chaindb.ErrKeyNotFound)
if !nodeHashDeletedInAnotherBlock {
return nil
} else if err != nil {
return fmt.Errorf("getting journal key for Merkle value from journal database: %w", err)
return fmt.Errorf("getting journal key for node hash from journal database: %w", err)
}

var key journalKey
Expand All @@ -231,7 +231,7 @@ func (p *FullNode) handleInsertedKey(insertedMerkleValue string, blockNumber uin
return nil
}

// Remove Merkle value from the deleted set of the block it was deleted in.
// Remove node hash from the deleted set of the block it was deleted in.
encodedJournalRecord, err := p.journalDatabase.Get(journalKeyDeletedAt)
if err != nil {
return fmt.Errorf("getting record from journal database: %w", err)
Expand All @@ -243,7 +243,7 @@ func (p *FullNode) handleInsertedKey(insertedMerkleValue string, blockNumber uin
return fmt.Errorf("decoding journal record: %w", err)
}

delete(record.DeletedMerkleValues, insertedMerkleValue)
delete(record.DeletedNodeHashes, insertedNodeHash)

encodedJournalRecord, err = scale.Marshal(record)
if err != nil {
Expand Down Expand Up @@ -322,8 +322,8 @@ func pruneStorage(blockNumber uint32, blockHashes []common.Hash,
return fmt.Errorf("getting journal record: %w", err)
}

for deletedMerkleValue := range record.DeletedMerkleValues {
err = batch.Del([]byte(deletedMerkleValue))
for deletedNodeHash := range record.DeletedNodeHashes {
err = batch.Del([]byte(deletedNodeHash))
if err != nil {
return fmt.Errorf("deleting key from batch: %w", err)
}
Expand Down Expand Up @@ -368,11 +368,11 @@ func storeJournalRecord(batch Putter, blockNumber uint32, blockHash common.Hash,
return fmt.Errorf("scale encoding journal key: %w", err)
}

for deletedMerkleValue := range record.DeletedMerkleValues {
for deletedNodeHash := range record.DeletedNodeHashes {
// We store the block hash + block number for each deleted node hash
// so a node hash can quickly be checked for from the journal database
// when running `handleInsertedKey`.
databaseKey := []byte(deletedMerkleValueKeyPrefix + deletedMerkleValue)
databaseKey := []byte(deletedNodeHashKeyPrefix + deletedNodeHash)
err = batch.Put(databaseKey, encodedKey)
if err != nil {
return fmt.Errorf("putting journal key in database batch: %w", err)
Expand Down
46 changes: 23 additions & 23 deletions internal/pruner/full_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ func Test_FullNode_pruneAll(t *testing.T) {
key := journalKey{BlockNumber: 1, BlockHash: common.Hash{2}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
}
encodedRecord := scaleMarshal(t, record)
database.EXPECT().Get(encodedKey).Return(encodedRecord, nil)
Expand Down Expand Up @@ -144,7 +144,7 @@ func Test_FullNode_pruneAll(t *testing.T) {
key := journalKey{BlockNumber: 1, BlockHash: common.Hash{2}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
}
encodedRecord := scaleMarshal(t, record)
database.EXPECT().Get(encodedKey).Return(encodedRecord, nil)
Expand Down Expand Up @@ -195,7 +195,7 @@ func Test_FullNode_pruneAll(t *testing.T) {
key := journalKey{BlockNumber: 1, BlockHash: common.Hash{2}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
}
encodedRecord := scaleMarshal(t, record)
database.EXPECT().Get(encodedKey).Return(encodedRecord, nil)
Expand Down Expand Up @@ -326,7 +326,7 @@ func Test_prune(t *testing.T) {
key := journalKey{BlockNumber: 1, BlockHash: common.Hash{2}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
}
encodedRecord := scaleMarshal(t, record)
database.EXPECT().Get(encodedKey).Return(encodedRecord, nil)
Expand Down Expand Up @@ -356,7 +356,7 @@ func Test_prune(t *testing.T) {
key := journalKey{BlockNumber: 1, BlockHash: common.Hash{2}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
}
encodedRecord := scaleMarshal(t, record)
database.EXPECT().Get(encodedKey).Return(encodedRecord, nil)
Expand Down Expand Up @@ -431,7 +431,7 @@ func Test_pruneStorage(t *testing.T) {
database := NewMockGetter(ctrl)
key := journalKey{BlockNumber: 10, BlockHash: common.Hash{1}}
encodedKey := scaleMarshal(t, key)
record := journalRecord{DeletedMerkleValues: map[string]struct{}{
record := journalRecord{DeletedNodeHashes: map[string]struct{}{
"deleted_hash": {},
}}
encodedRecord := scaleMarshal(t, record)
Expand All @@ -454,7 +454,7 @@ func Test_pruneStorage(t *testing.T) {

key1 := journalKey{BlockNumber: 10, BlockHash: common.Hash{1}}
encodedKey1 := scaleMarshal(t, key1)
record1 := journalRecord{DeletedMerkleValues: map[string]struct{}{
record1 := journalRecord{DeletedNodeHashes: map[string]struct{}{
"deleted_hash_1": {},
"deleted_hash_2": {},
}}
Expand All @@ -463,7 +463,7 @@ func Test_pruneStorage(t *testing.T) {

key2 := journalKey{BlockNumber: 10, BlockHash: common.Hash{2}}
encodedKey2 := scaleMarshal(t, key2)
record2 := journalRecord{DeletedMerkleValues: map[string]struct{}{
record2 := journalRecord{DeletedNodeHashes: map[string]struct{}{
"deleted_hash_3": {},
}}
encodedRecord2 := scaleMarshal(t, record2)
Expand Down Expand Up @@ -593,7 +593,7 @@ func Test_storeJournalRecord(t *testing.T) {
},
blockNumber: 1,
blockHash: common.Hash{2},
record: journalRecord{DeletedMerkleValues: map[string]struct{}{"deletedhash": {}}},
record: journalRecord{DeletedNodeHashes: map[string]struct{}{"deletedhash": {}}},
errWrapped: errTest,
errMessage: "putting journal key in database batch: test error",
},
Expand All @@ -604,17 +604,17 @@ func Test_storeJournalRecord(t *testing.T) {
encodedKey := scaleMarshal(t, journalKey{BlockNumber: 1, BlockHash: common.Hash{2}})
database.EXPECT().Put(databaseKey, encodedKey).Return(nil)
encodedRecord := scaleMarshal(t, journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
InsertedMerkleValues: map[string]struct{}{"insertedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
InsertedNodeHashes: map[string]struct{}{"insertedhash": {}},
})
database.EXPECT().Put(encodedKey, encodedRecord).Return(errTest)
return database
},
blockNumber: 1,
blockHash: common.Hash{2},
record: journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
InsertedMerkleValues: map[string]struct{}{"insertedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
InsertedNodeHashes: map[string]struct{}{"insertedhash": {}},
},
errWrapped: errTest,
errMessage: "putting journal record in database batch: test error",
Expand All @@ -626,17 +626,17 @@ func Test_storeJournalRecord(t *testing.T) {
encodedKey := scaleMarshal(t, journalKey{BlockNumber: 1, BlockHash: common.Hash{2}})
database.EXPECT().Put(databaseKey, encodedKey).Return(nil)
encodedRecord := scaleMarshal(t, journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
InsertedMerkleValues: map[string]struct{}{"insertedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
InsertedNodeHashes: map[string]struct{}{"insertedhash": {}},
})
database.EXPECT().Put(encodedKey, encodedRecord).Return(nil)
return database
},
blockNumber: 1,
blockHash: common.Hash{2},
record: journalRecord{
DeletedMerkleValues: map[string]struct{}{"deletedhash": {}},
InsertedMerkleValues: map[string]struct{}{"insertedhash": {}},
DeletedNodeHashes: map[string]struct{}{"deletedhash": {}},
InsertedNodeHashes: map[string]struct{}{"insertedhash": {}},
},
},
}
Expand Down Expand Up @@ -681,7 +681,7 @@ func Test_getJournalRecord(t *testing.T) {
blockNumber: 1,
blockHash: common.Hash{2},
errWrapped: errTest,
errMessage: "getting journal record from database: test error",
errMessage: "getting from database: test error",
},
"scale decoding error": {
databaseBuilder: func(ctrl *gomock.Controller) Getter {
Expand All @@ -701,17 +701,17 @@ func Test_getJournalRecord(t *testing.T) {
database := NewMockGetter(ctrl)
expectedKey := scaleMarshal(t, journalKey{BlockNumber: 1, BlockHash: common.Hash{2}})
returnedValue := scaleMarshal(t, journalRecord{
InsertedMerkleValues: map[string]struct{}{"a": {}, "b": {}},
DeletedMerkleValues: map[string]struct{}{"b": {}, "c": {}},
InsertedNodeHashes: map[string]struct{}{"a": {}, "b": {}},
DeletedNodeHashes: map[string]struct{}{"b": {}, "c": {}},
})
database.EXPECT().Get(expectedKey).Return(returnedValue, nil)
return database
},
blockNumber: 1,
blockHash: common.Hash{2},
record: journalRecord{
InsertedMerkleValues: map[string]struct{}{"a": {}, "b": {}},
DeletedMerkleValues: map[string]struct{}{"b": {}, "c": {}},
InsertedNodeHashes: map[string]struct{}{"a": {}, "b": {}},
DeletedNodeHashes: map[string]struct{}{"b": {}, "c": {}},
},
},
}
Expand Down Expand Up @@ -758,7 +758,7 @@ func Test_storeBlockNumberAtKey(t *testing.T) {
key: []byte("key"),
blockNumber: 1,
errWrapped: errTest,
errMessage: "putting block number in database batch: test error",
errMessage: "putting block number 1: test error",
},
"success": {
batchBuilder: func(ctrl *gomock.Controller) Putter {
Expand Down

0 comments on commit febbb27

Please sign in to comment.