Skip to content

Commit

Permalink
interim commit, using Function as interface rather than PreImageStorage
Browse files Browse the repository at this point in the history
bonsai worldstate tests fixes and hacks
  -> getOrCreate in BonsaiWorldStateUpdateAccumulator - do not throw if we discover an empty account in a non-null BonsaiValue<Account>
  -> add curentStateRoot to t8n
  -> preImageHasher function in BonsaiWorldState / Accumulator
    .. hacky setting/copying of preImageHasher on construction/copying just in Layered.  if we keep this route, we need to do it for snapshots also
  -> storageEntriesFrom and streamAccounts implemented in BonsaiWorldStateKeyValueStorage

Signed-off-by: garyschulte <garyschulte@gmail.com>
  • Loading branch information
garyschulte committed Aug 4, 2023
1 parent 05a2e0e commit 1618f33
Show file tree
Hide file tree
Showing 36 changed files with 206 additions and 69 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.Optional;
import java.util.TreeMap;
import java.util.stream.Collectors;

import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
Expand Down Expand Up @@ -220,7 +223,7 @@ public UInt256 getOriginalStorageValue(final UInt256 key) {
@Override
public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(
final Bytes32 startKeyHash, final int limit) {
throw new RuntimeException("Bonsai Tries does not currently support enumerating storage");
return context.getWorldStateStorage().storageEntriesFrom(this.addressHash, startKeyHash, limit);
}

public Bytes serializeAccount() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
package org.hyperledger.besu.ethereum.bonsai.storage;public interface BonsaiPreImageProxy {
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.CODE_STORAGE;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;

import org.apache.tuweni.units.bigints.UInt256;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
import org.hyperledger.besu.ethereum.bonsai.storage.flat.FlatDbReaderStrategy;
Expand All @@ -31,6 +32,7 @@
import org.hyperledger.besu.ethereum.worldstate.FlatDbMode;
import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.account.AccountStorageEntry;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
Expand All @@ -41,10 +43,14 @@
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;

import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
Expand All @@ -53,7 +59,7 @@

@SuppressWarnings("unused")
public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoCloseable {

Bytes32 BYTES32_MAX_VALUE = Bytes32.fromHexString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
private static final Logger LOG = LoggerFactory.getLogger(BonsaiWorldStateKeyValueStorage.class);

// 0x776f726c64526f6f74
Expand All @@ -79,6 +85,9 @@ public class BonsaiWorldStateKeyValueStorage implements WorldStateStorage, AutoC

protected final Subscribers<BonsaiStorageSubscriber> subscribers = Subscribers.create();

// no-op default hash preImage mapper:
protected Function<Hash, Optional<Bytes>> hashPreImageMapper = __ -> Optional.empty();

public BonsaiWorldStateKeyValueStorage(
final StorageProvider provider, final ObservableMetricsSystem metricsSystem) {
this.composedWorldStateStorage =
Expand Down Expand Up @@ -258,6 +267,33 @@ public Map<Bytes32, Bytes> streamFlatStorages(
composedWorldStateStorage, accountHash, startKeyHash, endKeyHash, max);
}

/**
* Provide a function to map hash values to an Optional wrapping their corresponding preImage
* or empty if the preImage is not available.
*
* @param hashPreImageMapper preImage mapper function
*/
public void setPreImageMapper(final Function<Hash, Optional<Bytes>> hashPreImageMapper) {
this.hashPreImageMapper = hashPreImageMapper;
}

public NavigableMap<Bytes32, AccountStorageEntry> storageEntriesFrom(final Hash addressHash,
final Bytes32 startKeyHash, final int limit) {
return streamFlatStorages(addressHash, startKeyHash, BYTES32_MAX_VALUE, limit)
.entrySet()
.stream()
.collect(Collectors.toMap(
e -> e.getKey(),
e -> AccountStorageEntry.create(
UInt256.fromBytes(e.getValue()),
Hash.wrap(e.getKey()),
hashPreImageMapper.apply(Hash.wrap(e.getKey()))
.map(UInt256::fromBytes)),
(a,b) -> a,
TreeMap::new
));
}

@Override
public Optional<Bytes> getNodeData(final Bytes location, final Bytes32 hash) {
return Optional.empty();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public BonsaiWorldStateLayerStorage(
final BonsaiWorldStateKeyValueStorage parent,
final ObservableMetricsSystem metricsSystem) {
super(parent, composedWorldStateStorage, trieLogStorage, metricsSystem);
setPreImageMapper(parent.hashPreImageMapper);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,9 @@

package org.hyperledger.besu.ethereum.bonsai.worldview;

import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP;
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY;
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;

import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;
import org.hyperledger.besu.datatypes.Address;
import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.datatypes.StorageSlotKey;
Expand All @@ -47,19 +45,20 @@
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.plugin.services.storage.SegmentIdentifier;
import org.hyperledger.besu.plugin.services.storage.SegmentedKeyValueStorageTransaction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;
import javax.annotation.Nonnull;

import org.apache.tuweni.bytes.Bytes;
import org.apache.tuweni.bytes.Bytes32;
import org.apache.tuweni.units.bigints.UInt256;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.hyperledger.besu.ethereum.bonsai.BonsaiAccount.fromRLP;
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_BLOCK_HASH_KEY;
import static org.hyperledger.besu.ethereum.bonsai.storage.BonsaiWorldStateKeyValueStorage.WORLD_ROOT_HASH_KEY;
import static org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE;

public class BonsaiWorldState
implements MutableWorldState, BonsaiWorldView, BonsaiStorageSubscriber {
Expand All @@ -68,8 +67,8 @@ public class BonsaiWorldState

public BonsaiWorldStateKeyValueStorage worldStateStorage;

private final CachedMerkleTrieLoader cachedMerkleTrieLoader;
private final TrieLogManager trieLogManager;
protected final CachedMerkleTrieLoader cachedMerkleTrieLoader;
protected final TrieLogManager trieLogManager;
private final BonsaiWorldStateUpdateAccumulator accumulator;

public Hash worldStateRootHash;
Expand All @@ -80,13 +79,14 @@ public class BonsaiWorldState
public BonsaiWorldState(
final BonsaiWorldStateProvider archive,
final BonsaiWorldStateKeyValueStorage worldStateStorage) {
this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager());
this(worldStateStorage, archive.getCachedMerkleTrieLoader(), archive.getTrieLogManager(), Hash::hash);
}

protected BonsaiWorldState(
final BonsaiWorldStateKeyValueStorage worldStateStorage,
final CachedMerkleTrieLoader cachedMerkleTrieLoader,
final TrieLogManager trieLogManager) {
final TrieLogManager trieLogManager,
final Function<Bytes, Hash> preImageHasher) {
this.worldStateStorage = worldStateStorage;
this.worldStateRootHash =
Hash.wrap(
Expand All @@ -100,7 +100,8 @@ protected BonsaiWorldState(
cachedMerkleTrieLoader.preLoadAccount(
getWorldStateStorage(), worldStateRootHash, addr),
(addr, value) ->
cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value));
cachedMerkleTrieLoader.preLoadStorageSlot(getWorldStateStorage(), addr, value),
preImageHasher);
this.cachedMerkleTrieLoader = cachedMerkleTrieLoader;
this.trieLogManager = trieLogManager;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ public class BonsaiWorldStateUpdateAccumulator
private final Map<Address, BonsaiValue<Bytes>> codeToUpdate = new ConcurrentHashMap<>();
private final Set<Address> storageToClear = Collections.synchronizedSet(new HashSet<>());

private final Function<Bytes, Hash> preImageHasher;

// storage sub mapped by _hashed_ key. This is because in self_destruct calls we need to
// enumerate the old storage and delete it. Those are trie stored by hashed key by spec and the
// alternative was to keep a giant pre-image cache of the entire trie.
Expand All @@ -78,11 +80,21 @@ public BonsaiWorldStateUpdateAccumulator(
final BonsaiWorldView world,
final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader,
final Consumer<StorageSlotKey> storagePreloader) {
super(world);
this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader);
this.accountPreloader = accountPreloader;
this.storagePreloader = storagePreloader;
this.isAccumulatorStateChanged = false;
// create accumulator without a preImage-aware Hash function:
this(world, accountPreloader, storagePreloader, Hash::hash);
}

protected BonsaiWorldStateUpdateAccumulator(
final BonsaiWorldView world,
final Consumer<BonsaiValue<BonsaiAccount>> accountPreloader,
final Consumer<StorageSlotKey> storagePreloader,
final Function<Bytes, Hash> preImageHasher) {
super(world);
this.accountsToUpdate = new AccountConsumingMap<>(new ConcurrentHashMap<>(), accountPreloader);
this.accountPreloader = accountPreloader;
this.storagePreloader = storagePreloader;
this.isAccumulatorStateChanged = false;
this.preImageHasher = preImageHasher;
}

public BonsaiWorldStateUpdateAccumulator copy() {
Expand Down Expand Up @@ -127,14 +139,18 @@ public EvmAccount createAccount(final Address address, final long nonce, final W
bonsaiValue = new BonsaiValue<>(null, null);
accountsToUpdate.put(address, bonsaiValue);
} else if (bonsaiValue.getUpdated() != null) {
throw new IllegalStateException("Cannot create an account when one already exists");
if (bonsaiValue.getUpdated().isEmpty()) {
return new WrappedEvmAccount(track(new UpdateTrackingAccount<>(bonsaiValue.getUpdated())));
} else {
throw new IllegalStateException("Cannot create an account when one already exists");
}
}

final BonsaiAccount newAccount =
new BonsaiAccount(
this,
address,
Hash.hash(address),
preImageHasher.apply(address),
nonce,
balance,
Hash.EMPTY_TRIE_HASH,
Expand Down Expand Up @@ -357,7 +373,7 @@ public void commit() {
entries.forEach(
storageUpdate -> {
final UInt256 keyUInt = storageUpdate.getKey();
final Hash slotHash = Hash.hash(keyUInt);
final Hash slotHash = preImageHasher.apply(keyUInt);
final StorageSlotKey slotKey =
new StorageSlotKey(slotHash, Optional.of(keyUInt));
final UInt256 value = storageUpdate.getValue();
Expand Down Expand Up @@ -401,7 +417,7 @@ public Optional<Bytes> getCode(final Address address, final Hash codeHash) {

@Override
public UInt256 getStorageValue(final Address address, final UInt256 slotKey) {
StorageSlotKey storageSlotKey = new StorageSlotKey(Hash.hash(slotKey), Optional.of(slotKey));
StorageSlotKey storageSlotKey = new StorageSlotKey(preImageHasher.apply(slotKey), Optional.of(slotKey));
return getStorageValueByStorageSlotKey(address, storageSlotKey).orElse(UInt256.ZERO);
}

Expand Down Expand Up @@ -446,7 +462,7 @@ public Optional<UInt256> getStorageValueByStorageSlotKey(
public UInt256 getPriorStorageValue(final Address address, final UInt256 storageKey) {
// TODO maybe log the read into the trie layer?
StorageSlotKey storageSlotKey =
new StorageSlotKey(Hash.hash(storageKey), Optional.of(storageKey));
new StorageSlotKey(preImageHasher.apply(storageKey), Optional.of(storageKey));
final Map<StorageSlotKey, BonsaiValue<UInt256>> localAccountStorage =
storageToUpdate.get(address);
if (localAccountStorage != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,9 @@ public void accountCreatedWhenBlockRewardIsZeroAndNotSkipped() {
final BlockHeader emptyBlockHeader =
new BlockHeaderTestFixture()
.transactionsRoot(Hash.EMPTY_LIST_HASH)
.stateRoot(Hash.fromHexString("0xa6b5d50f7b3c39b969c2fe8fed091939c674fef49b4826309cb6994361e39b71"))
.stateRoot(
Hash.fromHexString(
"0xa6b5d50f7b3c39b969c2fe8fed091939c674fef49b4826309cb6994361e39b71"))
.ommersHash(Hash.EMPTY_LIST_HASH)
.buildHeader();
blockProcessor.processBlock(blockchain, worldState, emptyBlockHeader, emptyList(), emptyList());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState;
import org.hyperledger.besu.ethereum.referencetests.DefaultReferenceTestWorldState;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

Expand Down Expand Up @@ -149,7 +149,10 @@ public void setup() {
return new BlockProcessingResult(
Optional.of(
new BlockProcessingOutputs(
new ReferenceTestWorldState(), blockDataGenerator.receipts(block))));
// use forest-based worldstate since it does not require
// blockheader stateroot to match actual worldstate root
DefaultReferenceTestWorldState.create(Collections.emptyMap()),
blockDataGenerator.receipts(block))));
});

backwardChain = inMemoryBackwardChain();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,10 @@
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.log.Log;
import org.hyperledger.besu.evm.tracing.OperationTracer;
import org.hyperledger.besu.evm.tracing.StandardJsonTracer;
import org.hyperledger.besu.evm.worldstate.WorldState;
import org.hyperledger.besu.evm.worldstate.WorldUpdater;
import org.hyperledger.besu.evmtool.exception.UnsupportedForkException;
import org.hyperledger.besu.util.LogConfigurator;
Expand Down Expand Up @@ -171,9 +169,7 @@ private void traceTestSpecs(final String test, final List<GeneralStateTestCaseEi
for (final GeneralStateTestCaseEipSpec spec : specs) {

final BlockHeader blockHeader = spec.getBlockHeader();
final WorldState initialWorldState = spec.getInitialWorldState();
final Transaction transaction = spec.getTransaction();

final ObjectNode summaryLine = objectMapper.createObjectNode();
if (transaction == null) {
if (parentCommand.showJsonAlloc || parentCommand.showJsonResults) {
Expand All @@ -188,7 +184,7 @@ private void traceTestSpecs(final String test, final List<GeneralStateTestCaseEi
summaryLine.put("pass", spec.getExpectException() != null);
summaryLine.put("validationError", "Transaction had out-of-bounds parameters");
} else {
final MutableWorldState worldState = new DefaultMutableWorldState(initialWorldState);
final MutableWorldState worldState = spec.getInitialWorldState().copy();
// Several of the GeneralStateTests check if the transaction could potentially
// consume more gas than is left for the block it's attempted to be included in.
// This check is performed within the `BlockImporter` rather than inside the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestBlockchain;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPInput;
import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
import org.hyperledger.besu.ethereum.rlp.RLP;
import org.hyperledger.besu.ethereum.worldstate.DefaultMutableWorldState;
import org.hyperledger.besu.evm.AccessListEntry;
import org.hyperledger.besu.evm.account.Account;
import org.hyperledger.besu.evm.account.AccountStorageEntry;
Expand Down Expand Up @@ -229,7 +229,7 @@ static T8nResult runTest(
final String rewardString,
final ObjectMapper objectMapper,
final ReferenceTestEnv referenceTestEnv,
final MutableWorldState initialWorldState,
final ReferenceTestWorldState initialWorldState,
final List<Transaction> transactions,
final List<RejectedTransaction> rejections,
final TracerManager tracerManager) {
Expand All @@ -238,7 +238,7 @@ static T8nResult runTest(
ReferenceTestProtocolSchedules.create(
new StubGenesisConfigOptions().chainId(BigInteger.valueOf(chainId)));

final MutableWorldState worldState = new DefaultMutableWorldState(initialWorldState);
final MutableWorldState worldState = initialWorldState.copy();

final ProtocolSchedule protocolSchedule = referenceTestProtocolSchedules.getByName(fork);
if (protocolSchedule == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import static org.hyperledger.besu.evmtool.T8nSubCommand.COMMAND_NAME;

import org.hyperledger.besu.datatypes.Hash;
import org.hyperledger.besu.ethereum.core.MutableWorldState;
import org.hyperledger.besu.ethereum.core.Transaction;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv;
import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState;
Expand Down Expand Up @@ -172,7 +171,7 @@ public void run() {
final ObjectMapper objectMapper = JsonUtils.createObjectMapper();
final ObjectReader t8nReader = objectMapper.reader();

MutableWorldState initialWorldState;
ReferenceTestWorldState initialWorldState;
ReferenceTestEnv referenceTestEnv;
List<Transaction> transactions = new ArrayList<>();
List<RejectedTransaction> rejections = new ArrayList<>();
Expand Down
Loading

0 comments on commit 1618f33

Please sign in to comment.