Skip to content

Commit

Permalink
Bonsai based reference test worldstate (hyperledger#5686)
Browse files Browse the repository at this point in the history
* create a bonsai based reference test worldstate
  -> getOrCreate in BonsaiWorldStateUpdateAccumulator - do not throw if we discover an empty account in a non-null BonsaiValue<Account>
  -> add curentStateRoot to t8n
  -> storageEntriesFrom and streamAccounts implemented in BonsaiWorldStateKeyValueStorage
  -> add endKey version of streamFromKey
* bonsai fix for self-destruct and create2 at the same address and same block

Signed-off-by: garyschulte <garyschulte@gmail.com>
Signed-off-by: Karim TAAM <karim.t2am@gmail.com>
Co-authored-by: Karim TAAM <karim.t2am@gmail.com>
  • Loading branch information
garyschulte and matkt committed Sep 20, 2023
1 parent 1221b2e commit 68d1562
Show file tree
Hide file tree
Showing 53 changed files with 746 additions and 141 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ public UInt256 getOriginalStorageValue(final UInt256 key) {
@Override
public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(
final Bytes32 startKeyHash, final int limit) {
throw new RuntimeException("Bonsai Tries does not currently support enumerating storage");
return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit);
}

public Bytes serializeAccount() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,8 @@ public BonsaiWorldStateProvider(
pluginContext);
this.blockchain = blockchain;
this.worldStateStorage = worldStateStorage;
this.persistedState = new BonsaiWorldState(this, worldStateStorage);
this.cachedMerkleTrieLoader = cachedMerkleTrieLoader;
this.persistedState = new BonsaiWorldState(this, worldStateStorage);
blockchain
.getBlockHeader(persistedState.getWorldStateBlockHash())
.ifPresent(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
/*
* Copyright Hyperledger Besu Contributors.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
*
*/
package org.hyperledger.besu.ethereum.bonsai.storage;

import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.worldstate.WorldStatePreimageStorage;

import java.util.Optional;

import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;

/** Acts as both a Hasher and PreImageStorage for Bonsai storage format. */
public interface BonsaiPreImageProxy extends WorldStatePreimageStorage {
/**
* If this value is not already present, save in preImage store and return the hash value.
*
* @param value value to hash
* @return Hash of value
*/
Hash hashAndSavePreImage(Bytes value);

/**
* A caching PreImageProxy suitable for ReferenceTestWorldState which saves hashes in an unbounded
* BiMap.
*/
class BonsaiReferenceTestPreImageProxy implements BonsaiPreImageProxy {
BiMap<Hash, Bytes> preImageCache = HashBiMap.create();

@Override
public synchronized Hash hashAndSavePreImage(final Bytes value) {
return preImageCache.inverse().computeIfAbsent(value, Hash::hash);
}

@Override
public Optional<UInt256> getStorageTrieKeyPreimage(final Bytes32 trieKey) {
return Optional.ofNullable(preImageCache.get(trieKey)).map(UInt256::fromBytes);
}

@Override
public Optional<Address> getAccountTrieKeyPreimage(final Bytes32 trieKey) {
return Optional.ofNullable(preImageCache.get(trieKey)).map(Address::wrap);
}

@Override
public Updater updater() {
throw new UnsupportedOperationException(
"BonsaiReferenceTestPreImageProxy does not implement an updater");
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.account.AccountStorageEntry;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
Expand All @@ -41,6 +42,7 @@
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
Expand All @@ -53,7 +55,6 @@

@SuppressWarnings("unused")
public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable {

private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class);

// 0x776f726c64526f6f74
Expand Down Expand Up @@ -250,6 +251,11 @@ public Map<Bytes32, Bytes> streamFlatStorages(
composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max);
}

public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(
final Hash addressHash, final Bytes32 startKeyHash, final int limit) {
throw new RuntimeException("Bonsai Tries does not currently support enumerating storage");
}

@Override
public Optional<Bytes> getNodeData(final Bytes location, final Bytes32 hash) {
return Optional.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -135,10 +135,10 @@ public Map<Bytes32, Bytes> streamAccountFlatDatabase(
final long max) {
final Stream<Pair<Bytes32, Bytes>> pairStream =
storage
.streamFromKey(ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe())
.streamFromKey(
ACCOUNT_INFO_STATE, startKeyHash.toArrayUnsafe(), endKeyHash.toArrayUnsafe())
.limit(max)
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())))
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0);
.map(pair -> new Pair<>(Bytes32.wrap(pair.getKey()), Bytes.wrap(pair.getValue())));

final TreeMap<Bytes32, Bytes> collected =
pairStream.collect(
Expand All @@ -157,15 +157,14 @@ public Map<Bytes32, Bytes> streamStorageFlatDatabase(
storage
.streamFromKey(
ACCOUNT_STORAGE_STORAGE,
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe())
.takeWhile(pair -> Bytes.wrap(pair.getKey()).slice(0, Hash.SIZE).equals(accountHash))
Bytes.concatenate(accountHash, startKeyHash).toArrayUnsafe(),
Bytes.concatenate(accountHash, endKeyHash).toArrayUnsafe())
.limit(max)
.map(
pair ->
new Pair<>(
Bytes32.wrap(Bytes.wrap(pair.getKey()).slice(Hash.SIZE)),
RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros())))
.takeWhile(pair -> pair.getFirst().compareTo(endKeyHash) <= 0);
RLP.encodeValue(Bytes.wrap(pair.getValue()).trimLeadingZeros())));

final TreeMap<Bytes32, Bytes> collected =
pairStream.collect(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.hyperledger.besu.ethereum.bonsai.BonsaiAccount;
import org.hyperledger.besu.ethereum.bonsai.BonsaiValue;
import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateProvider;
import org.hyperledger.besu.ethereum.bonsai.cache.CachedMerkleTrieLoader;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiSnapshotWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage;
import org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber;
Expand Down Expand Up @@ -65,51 +66,52 @@ public class BonsaiWorldState

private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldState.class);

private BonsaiWorldStateKeyValueStorage worldStateStorage;
protected BonsaiWorldStateKeyValueStorage worldStateStorage;

private final BonsaiWorldStateProvider archive;
private final BonsaiWorldStateUpdateAccumulator accumulator;
protected final CachedMerkleTrieLoader cachedMerkleTrieLoader;
protected final TrieLogManager trieLogManager;
private BonsaiWorldStateUpdateAccumulator accumulator;

private Hash worldStateRootHash;
protected Hash worldStateRootHash;
Hash worldStateBlockHash;

private boolean isFrozen;

public BonsaiWorldState(
final BonsaiWorldStateProvider archive,
final BonsaiWorldStateKeyValueStorage worldStateStorage) {
this.archive = archive;
this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager());
}

protected BonsaiWorldState(
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final TrieLogManager trieLogManager) {
this.worldStateStorage = worldStateStorage;
worldStateRootHash =
this.worldStateRootHash =
Hash.wrap(
Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH)));
worldStateBlockHash =
this.worldStateBlockHash =
Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO)));
accumulator =
this.accumulator =
new BonsaiWorldStateUpdateAccumulator(
this,
(addr, value) ->
archive
.getCachedMerkleTrieLoader()
.preLoadAccount(getWorldStateStorage(), worldStateRootHash, addr),
cachedMerkleTrieLoader.preLoadAccount(
getWorldStateStorage(), worldStateRootHash, addr),
(addr, value) ->
archive
.getCachedMerkleTrieLoader()
.preLoadStorageSlot(getWorldStateStorage(), addr, value));
cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value));
this.cachedMerkleTrieLoader = cachedMerkleTrieLoader;
this.trieLogManager = trieLogManager;
}

public BonsaiWorldState(
final BonsaiWorldStateProvider archive,
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final BonsaiWorldStateUpdateAccumulator updater) {
this.archive = archive;
this.worldStateStorage = worldStateStorage;
this.worldStateRootHash =
Hash.wrap(
Bytes32.wrap(worldStateStorage.getWorldStateRootHash().orElse(Hash.EMPTY_TRIE_HASH)));
this.worldStateBlockHash =
Hash.wrap(Bytes32.wrap(worldStateStorage.getWorldStateBlockHash().orElse(Hash.ZERO)));
this.accumulator = updater;
/**
* Having a protected method to override the accumulator solves the chicken-egg problem of needing
* a worldstate reference (this) when construction the Accumulator.
*
* @param accumulator accumulator to use.
*/
protected void setAccumulator(final BonsaiWorldStateUpdateAccumulator accumulator) {
this.accumulator = accumulator;
}

/**
Expand All @@ -130,10 +132,6 @@ public Hash getWorldStateRootHash() {
return worldStateRootHash;
}

public BonsaiWorldStateProvider getArchive() {
return archive;
}

@Override
public boolean isPersisted() {
return isPersisted(worldStateStorage);
Expand Down Expand Up @@ -189,9 +187,7 @@ private Hash calculateRootHash(
final StoredMerklePatriciaTrie<Bytes, Bytes> accountTrie =
createTrie(
(location, hash) ->
archive
.getCachedMerkleTrieLoader()
.getAccountStateTrieNode(worldStateStorage, location, hash),
cachedMerkleTrieLoader.getAccountStateTrieNode(worldStateStorage, location, hash),
worldStateRootHash);

// for manicured tries and composting, collect branches here (not implemented)
Expand Down Expand Up @@ -223,7 +219,7 @@ private void updateTheAccounts(
final BonsaiAccount updatedAccount = bonsaiValue.getUpdated();
try {
if (updatedAccount == null) {
final Hash addressHash = Hash.hash(accountKey);
final Hash addressHash = hashAndSavePreImage(accountKey);
accountTrie.remove(addressHash);
maybeStateUpdater.ifPresent(
bonsaiUpdater -> bonsaiUpdater.removeAccountInfoState(addressHash));
Expand All @@ -232,7 +228,7 @@ private void updateTheAccounts(
final Bytes accountValue = updatedAccount.serializeAccount();
maybeStateUpdater.ifPresent(
bonsaiUpdater ->
bonsaiUpdater.putAccountInfoState(Hash.hash(accountKey), accountValue));
bonsaiUpdater.putAccountInfoState(hashAndSavePreImage(accountKey), accountValue));
accountTrie.put(addressHash, accountValue);
}
} catch (MerkleTrieException e) {
Expand Down Expand Up @@ -277,10 +273,8 @@ private void updateAccountStorageState(
final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie =
createTrie(
(location, key) ->
archive
.getCachedMerkleTrieLoader()
.getAccountStorageTrieNode(
worldStateStorage, updatedAddressHash, location, key),
cachedMerkleTrieLoader.getAccountStorageTrieNode(
worldStateStorage, updatedAddressHash, location, key),
storageRoot);

// for manicured tries and composting, collect branches here (not implemented)
Expand Down Expand Up @@ -405,7 +399,6 @@ public void persist(final BlockHeader blockHeader) {
}
saveTrieLog =
() -> {
final TrieLogManager trieLogManager = archive.getTrieLogManager();
trieLogManager.saveTrieLog(localCopy, newWorldStateRootHash, blockHeader, this);
// not save a frozen state in the cache
if (!isFrozen) {
Expand Down Expand Up @@ -626,4 +619,9 @@ private void closeFrozenStorage() {
// no op
}
}

protected Hash hashAndSavePreImage(final Bytes value) {
// by default do not save has preImages
return Hash.hash(value);
}
}
Loading

0 comments on commit 68d1562

Please sign in to comment.