diff --git a/lib/modAvmVersion2.jar b/lib/modAvmVersion2.jar index bb8e7bcc72..ee55317f70 100644 Binary files a/lib/modAvmVersion2.jar and b/lib/modAvmVersion2.jar differ diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionContractDetailsImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionContractDetailsImpl.java index a018045576..e63e492035 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionContractDetailsImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionContractDetailsImpl.java @@ -34,11 +34,9 @@ public class AionContractDetailsImpl implements ContractDetails { private boolean dirty = false; private boolean deleted = false; - // a value > 0 indicates that prune should be for that many blocks. - private int prune = 0; - // indicates the maximum storage size before shifting to the storage database - // NOTE: updating this value can lead to incompatible data storage - private int detailsInMemoryStorageLimit = 64 * 1024; + /** Indicates the maximum storage size before shifting to the storage database. */ + @VisibleForTesting + static int detailsInMemoryStorageLimit = 64 * 1024; private Map codes = new HashMap<>(); // classes extending this rely on this value starting off as null @@ -50,28 +48,17 @@ public class AionContractDetailsImpl implements ContractDetails { private ByteArrayKeyValueStore dataSource; private ByteArrayKeyValueStore objectGraphSource = null; - private byte[] rlpEncoded; - private AionAddress address; private SecureTrie storageTrie = new SecureTrie(null); - public boolean externalStorage; - private ByteArrayKeyValueStore externalStorageDataSource; - private ByteArrayKeyValueStore contractObjectGraphSource = null; + private boolean externalStorage; private byte[] objectGraphHash = EMPTY_DATA_HASH; private byte[] concatenatedStorageHash = EMPTY_DATA_HASH; public AionContractDetailsImpl() {} - @VisibleForTesting - AionContractDetailsImpl(int prune, int memStorageLimit) { - this.prune = prune; - // NOTE: updating this value can lead to incompatible data storage - this.detailsInMemoryStorageLimit = memStorageLimit; - } - /** * Creates a object with attached database access for the storage and object graph. * @@ -102,6 +89,11 @@ public AionContractDetailsImpl(byte[] code) throws Exception { decode(code); } + @VisibleForTesting + public boolean isExternalStorage() { + return externalStorage; + } + @Override public byte[] getCode(byte[] codeHash) { if (java.util.Arrays.equals(codeHash, EMPTY_DATA_HASH)) { @@ -122,12 +114,15 @@ private void setCodes(Map codes) { @Override public void appendCodes(Map codes) { + if (!this.codes.keySet().containsAll(codes.keySet())) { + this.dirty = true; + } this.codes.putAll(codes); } @Override - public void setDirty(boolean dirty) { - this.dirty = dirty; + public void markAsDirty() { + this.dirty = true; } @Override @@ -136,6 +131,7 @@ public boolean isDirty() { } public void delete() { + // TODO: should we set dirty=true? this.deleted = true; } @@ -185,8 +181,7 @@ public void put(ByteArrayWrapper key, ByteArrayWrapper value) { byte[] data = RLP.encodeElement(value.toBytes()); storageTrie.update(key.toBytes(), data); - setDirty(true); - rlpEncoded = null; + dirty = true; } @Override @@ -195,8 +190,7 @@ public void delete(ByteArrayWrapper key) { storageTrie.delete(key.toBytes()); - setDirty(true); - rlpEncoded = null; + dirty = true; } /** @@ -237,8 +231,7 @@ public void setCode(byte[] code) { e.printStackTrace(); return; } - setDirty(true); - rlpEncoded = null; + dirty = true; } @Override @@ -264,8 +257,7 @@ public void setObjectGraph(byte[] graph) { this.objectGraph = graph; this.objectGraphHash = h256(objectGraph); - this.setDirty(true); - this.rlpEncoded = null; + dirty = true; } /** @@ -305,39 +297,23 @@ private byte[] computeAvmStorageHash() { */ public void decode(byte[] rlpCode) { // TODO: remove vm type requirement when refactoring into separate AVM & FVM implementations - decode(rlpCode, false); - } - - /** - * Decodes an AionContractDetailsImpl object from the RLP encoding rlpCode. The fast check flag - * indicates whether the contractDetails needs to sync with external storage. - * - * @param rlpCode The encoding to decode. - * @param fastCheck indicates whether the contractDetails needs to sync with external storage. - */ - public void decode(byte[] rlpCode, boolean fastCheck) { RLPList data = RLP.decode2(rlpCode); RLPList rlpList = (RLPList) data.get(0); // partial decode either encoding - boolean keepStorageInMem = decodeEncodingWithoutVmType(rlpList, fastCheck); + boolean keepStorageInMem = decodeEncodingWithoutVmType(rlpList); if (rlpList.size() != 5) { // revert back from storing the VM type in details // force a save with new encoding throw new IllegalStateException( "Incompatible data storage. Please shutdown the kernel and perform database migration to version 1.0 (Denali) of the kernel as instructed in the release."); - } else { - // keep encoding when compatible with new style - this.rlpEncoded = rlpCode; } - if (!fastCheck || externalStorage || !keepStorageInMem) { // it was not a fast check - // NOTE: under normal circumstances the VM type is set by the details data store - // Do not forget to set the vmType value externally during tests!!! - decodeStorage(rlpList.get(2), rlpList.get(3), keepStorageInMem); - } + // NOTE: under normal circumstances the VM type is set by the details data store + // Do not forget to set the vmType value externally during tests!!! + decodeStorage(rlpList.get(2), rlpList.get(3), keepStorageInMem); } /** @@ -351,17 +327,12 @@ public void decode(byte[] rlpCode, boolean fastCheck) { * @return {@code true} if the storage must continue to be kept in memory, {@code false} * otherwise */ - public boolean decodeEncodingWithoutVmType(RLPList rlpList, boolean fastCheck) { + public boolean decodeEncodingWithoutVmType(RLPList rlpList) { RLPItem isExternalStorage = (RLPItem) rlpList.get(1); RLPItem storage = (RLPItem) rlpList.get(3); this.externalStorage = isExternalStorage.getRLPData().length > 0; boolean keepStorageInMem = storage.getRLPData().length <= detailsInMemoryStorageLimit; - // No externalStorage require. - if (fastCheck && !externalStorage && keepStorageInMem) { - return keepStorageInMem; - } - RLPItem address = (RLPItem) rlpList.get(0); RLPElement code = rlpList.get(4); @@ -423,7 +394,6 @@ public void decodeStorage(RLPElement root, RLPElement storage, boolean keepStora } else { storageTrie.deserialize(storage.getRLPData()); } - storageTrie.withPruningEnabled(prune > 0); // switch from in-memory to external storage if (!externalStorage && !keepStorageInMem) { @@ -441,34 +411,24 @@ public void decodeStorage(RLPElement root, RLPElement storage, boolean keepStora * @return an rlp encoding of this. */ public byte[] getEncoded() { - if (rlpEncoded == null) { - - byte[] rlpAddress = RLP.encodeElement(address.toByteArray()); - byte[] rlpIsExternalStorage = RLP.encodeByte((byte) (externalStorage ? 1 : 0)); - byte[] rlpStorageRoot; - // encoding for AVM - if (vmType == InternalVmType.AVM) { - rlpStorageRoot = RLP.encodeElement(computeAvmStorageHash()); - } else { - rlpStorageRoot = - RLP.encodeElement( - externalStorage ? storageTrie.getRootHash() : EMPTY_BYTE_ARRAY); - } - byte[] rlpStorage = - RLP.encodeElement(externalStorage ? EMPTY_BYTE_ARRAY : storageTrie.serialize()); - byte[][] codes = new byte[getCodes().size()][]; - int i = 0; - for (byte[] bytes : this.getCodes().values()) { - codes[i++] = RLP.encodeElement(bytes); - } - byte[] rlpCode = RLP.encodeList(codes); - - this.rlpEncoded = - RLP.encodeList( - rlpAddress, rlpIsExternalStorage, rlpStorageRoot, rlpStorage, rlpCode); + byte[] rlpAddress = RLP.encodeElement(address.toByteArray()); + byte[] rlpIsExternalStorage = RLP.encodeByte((byte) (externalStorage ? 1 : 0)); + byte[] rlpStorageRoot; + // encoding for AVM + if (vmType == InternalVmType.AVM) { + rlpStorageRoot = RLP.encodeElement(computeAvmStorageHash()); + } else { + rlpStorageRoot = RLP.encodeElement(externalStorage ? storageTrie.getRootHash() : EMPTY_BYTE_ARRAY); } + byte[] rlpStorage = RLP.encodeElement(externalStorage ? EMPTY_BYTE_ARRAY : storageTrie.serialize()); + byte[][] codes = new byte[getCodes().size()][]; + int i = 0; + for (byte[] bytes : this.getCodes().values()) { + codes[i++] = RLP.encodeElement(bytes); + } + byte[] rlpCode = RLP.encodeList(codes); - return rlpEncoded; + return RLP.encodeList(rlpAddress, rlpIsExternalStorage, rlpStorageRoot, rlpStorage, rlpCode); } /** @@ -495,7 +455,6 @@ public void setAddress(AionAddress address) { if (ContractInfo.isPrecompiledContract(address)) { setVmType(InternalVmType.FVM); } - this.rlpEncoded = null; } /** Syncs the storage trie. */ @@ -529,15 +488,6 @@ public void syncStorage() { } } - /** - * Sets the storage data source. - * - * @param dataSource The new dataSource. - */ - public void setDataSource(ByteArrayKeyValueStore dataSource) { - this.dataSource = dataSource; - } - /** * Sets the data source for storing the AVM object graph. * @@ -553,12 +503,7 @@ public void setObjectGraphSource(ByteArrayKeyValueStore objectGraphSource) { * @return the external storage data source. */ private ByteArrayKeyValueStore getExternalStorageDataSource() { - if (externalStorageDataSource == null) { - externalStorageDataSource = - new XorDataSource( - dataSource, h256(("details-storage/" + address.toString()).getBytes())); - } - return externalStorageDataSource; + return new XorDataSource(dataSource, h256(("details-storage/" + address.toString()).getBytes())); } /** @@ -567,31 +512,18 @@ private ByteArrayKeyValueStore getExternalStorageDataSource() { * @return the data source specific to the current contract. */ private ByteArrayKeyValueStore getContractObjectGraphSource() { - if (contractObjectGraphSource == null) { - if (objectGraphSource == null) { - throw new NullPointerException( - "The contract object graph source was not initialized."); - } else { - contractObjectGraphSource = - new XorDataSource( - objectGraphSource, - h256(("details-graph/" + address.toString()).getBytes())); - } + if (objectGraphSource == null) { + throw new NullPointerException("The contract object graph source was not initialized."); + } else { + return new XorDataSource(objectGraphSource, h256(("details-graph/" + address.toString()).getBytes())); } - return contractObjectGraphSource; } /** * Sets the external storage data source to dataSource. - * - * @param dataSource The new data source. - * @implNote The tests are taking a shortcut here in bypassing the XorDataSource created by - * {@link #getExternalStorageDataSource()}. Do not use this method in production. */ @VisibleForTesting - void setExternalStorageDataSource(ByteArrayKeyValueStore dataSource) { - // TODO: regarding the node above: the tests should be updated and the method removed - this.externalStorageDataSource = dataSource; + void initializeExternalStorageTrieForTest() { this.externalStorage = true; this.storageTrie = new SecureTrie(getExternalStorageDataSource()); } @@ -663,7 +595,6 @@ public AionContractDetailsImpl getSnapshotTo(byte[] hash, InternalVmType vm) { // storage information details.externalStorage = this.externalStorage; - details.externalStorageDataSource = this.externalStorageDataSource; details.dataSource = dataSource; return details; @@ -692,12 +623,10 @@ public AionContractDetailsImpl copy() { // storage information aionContractDetailsCopy.dataSource = this.dataSource; - aionContractDetailsCopy.externalStorageDataSource = this.externalStorageDataSource; aionContractDetailsCopy.externalStorage = this.externalStorage; // object graph information aionContractDetailsCopy.objectGraphSource = this.objectGraphSource; - aionContractDetailsCopy.contractObjectGraphSource = this.contractObjectGraphSource; aionContractDetailsCopy.objectGraph = objectGraph == null ? null @@ -714,16 +643,10 @@ public AionContractDetailsImpl copy() { : Arrays.copyOf( this.concatenatedStorageHash, this.concatenatedStorageHash.length); - aionContractDetailsCopy.prune = this.prune; - aionContractDetailsCopy.detailsInMemoryStorageLimit = this.detailsInMemoryStorageLimit; aionContractDetailsCopy.setCodes(getDeepCopyOfCodes()); - aionContractDetailsCopy.setDirty(this.isDirty()); + aionContractDetailsCopy.dirty = this.dirty; aionContractDetailsCopy.deleted = this.deleted; aionContractDetailsCopy.address = this.address; - aionContractDetailsCopy.rlpEncoded = - (this.rlpEncoded == null) - ? null - : Arrays.copyOf(this.rlpEncoded, this.rlpEncoded.length); aionContractDetailsCopy.storageTrie = (this.storageTrie == null) ? null : this.storageTrie.copy(); return aionContractDetailsCopy; diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryCache.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryCache.java index ced775dd02..12d0343023 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryCache.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryCache.java @@ -63,8 +63,7 @@ public void createAccount(AionAddress address) { // TODO: unify contract details initialization from Impl and Track ContractDetails contractDetails = new ContractDetailsCacheImpl(null); - // TODO: refactor to use makeDirty() from AbstractState - contractDetails.setDirty(true); + contractDetails.markAsDirty(); cachedDetails.put(address, contractDetails); } finally { lock.unlock(); @@ -278,8 +277,6 @@ public void saveCode(AionAddress address, byte[] code) { // preexisting code! ContractDetails contractDetails = getContractDetails(address); contractDetails.setCode(code); - // TODO: ensure that setDirty is done by the class itself - contractDetails.setDirty(true); // update the code hash getAccountState(address).setCodeHash(h256(code)); diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 357f4b42b3..04eb9697c6 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -473,18 +473,16 @@ private void updateAccountState(AionAddress address, AccountState accountState) /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that the contract details - * returned is a newly created object by {@link AionContractDetailsImpl#getSnapshotTo(byte[], - * InternalVmType)}. Since this querying method it locked, the methods calling it may not - * need to be locked or synchronized, depending on the specific use case. + * @implNote Methods calling this can rely on the fact that the contract details returned is a + * newly created snapshot object. Since this method it locked, the methods using the + * returned object do not need to be locked or synchronized, depending on the + * specific use case. */ @Override public AionContractDetailsImpl getContractDetails(AionAddress address) { rwLock.readLock().lock(); try { - AionContractDetailsImpl details; - // That part is important cause if we have // to sync details storage according the trie root // saved in the account @@ -497,13 +495,7 @@ public AionContractDetailsImpl getContractDetails(AionAddress address) { } InternalVmType vm = getVMUsed(address, codeHash); - details = detailsDS.get(vm, address.toByteArray()); - - if (details != null) { - details = details.getSnapshotTo(storageRoot, vm); - } - - return details; + return detailsDS.getSnapshot(vm, address.toByteArray(), storageRoot); } finally { rwLock.readLock().unlock(); } diff --git a/modAionImpl/src/org/aion/zero/impl/db/ContractDetailsCacheImpl.java b/modAionImpl/src/org/aion/zero/impl/db/ContractDetailsCacheImpl.java index c0c634a14e..6c53c7c332 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/ContractDetailsCacheImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/ContractDetailsCacheImpl.java @@ -50,7 +50,7 @@ public static ContractDetailsCacheImpl copy(ContractDetailsCacheImpl cache) { copy.objectGraph = Arrays.copyOf(cache.objectGraph, cache.objectGraph.length); } copy.storage = new HashMap<>(cache.storage); - copy.setDirty(cache.isDirty()); + copy.dirty = cache.dirty; copy.deleted = cache.deleted; return copy; } @@ -75,7 +75,7 @@ public void setCode(byte[] code) { e.printStackTrace(); return; } - setDirty(true); + dirty = true; } @Override @@ -89,12 +89,15 @@ private void setCodes(Map codes) { @Override public void appendCodes(Map codes) { + if (!this.codes.keySet().containsAll(codes.keySet())) { + this.dirty = true; + } this.codes.putAll(codes); } @Override - public void setDirty(boolean dirty) { - this.dirty = dirty; + public void markAsDirty() { + this.dirty = true; } @Override @@ -104,6 +107,7 @@ public boolean isDirty() { @Override public void delete() { + // TODO: should we set dirty=true? this.deleted = true; } @@ -146,7 +150,7 @@ public void put(ByteArrayWrapper key, ByteArrayWrapper value) { Objects.requireNonNull(value); storage.put(key, value); - setDirty(true); + dirty = true; } @Override @@ -154,7 +158,7 @@ public void delete(ByteArrayWrapper key) { Objects.requireNonNull(key); storage.put(key, null); - setDirty(true); + dirty = true; } /** @@ -217,7 +221,7 @@ public void setObjectGraph(byte[] graph) { Objects.requireNonNull(graph); objectGraph = graph; - setDirty(true); + dirty = true; } /** @@ -293,8 +297,9 @@ public void commit() { } origContract.appendCodes(getCodes()); - - origContract.setDirty(this.isDirty() || origContract.isDirty()); + if (this.isDirty()) { + origContract.markAsDirty(); + } } /** @@ -332,7 +337,7 @@ public ContractDetailsCacheImpl copy() { } copy.storage = getDeepCopyOfStorage(); copy.setCodes(getDeepCopyOfCodes()); - copy.setDirty(this.isDirty()); + copy.dirty = this.dirty; copy.deleted = this.deleted; return copy; } diff --git a/modAionImpl/src/org/aion/zero/impl/db/DetailsDataStore.java b/modAionImpl/src/org/aion/zero/impl/db/DetailsDataStore.java index efb7d4d537..b1bae6a396 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/DetailsDataStore.java +++ b/modAionImpl/src/org/aion/zero/impl/db/DetailsDataStore.java @@ -33,28 +33,25 @@ public DetailsDataStore( } /** - * Fetches the ContractDetails from the cache, and if it doesn't exist, add to the remove set. + * Fetches the ContractDetails with the given root. * - * @param key the contract address as bytes * @param vm the virtual machine used at contract deployment - * @return + * @param key the contract address as bytes + * @param storageRoot the requested storage root + * @return a snapshot of the contract details with the requested root */ - public synchronized AionContractDetailsImpl get(InternalVmType vm, byte[] key) { - + public synchronized AionContractDetailsImpl getSnapshot(InternalVmType vm, byte[] key, byte[] storageRoot) { Optional rawDetails = detailsSrc.get(key); - // If it doesn't exist in cache or database. - if (!rawDetails.isPresent()) { + if (rawDetails.isPresent()) { + // decode raw details and return snapshot + AionContractDetailsImpl detailsImpl = new AionContractDetailsImpl(storageDSPrune, graphSrc); + detailsImpl.setVmType(vm); + detailsImpl.decode(rawDetails.get()); + return detailsImpl.getSnapshotTo(storageRoot, vm); + } else { return null; } - - // Found something from cache or database, return it by decoding it. - AionContractDetailsImpl detailsImpl = new AionContractDetailsImpl(storageDSPrune, graphSrc); - detailsImpl.setVmType(vm); - detailsImpl.decode(rawDetails.get()); // We can safely get as we checked - // if it is present. - - return detailsImpl; } /** Determine if the contract exists in the database. */ @@ -76,61 +73,9 @@ public synchronized void update(AionAddress key, AionContractDetailsImpl contrac } public synchronized void remove(byte[] key) { - ByteArrayWrapper wrappedKey = wrap(key); detailsSrc.delete(key); } - public synchronized void flush() { - flushInternal(); - } - - private long flushInternal() { - long totalSize = 0; - - syncLargeStorage(); - - // Get everything from the cache and calculate the size. - Iterator keysFromSource = detailsSrc.keys(); - while (keysFromSource.hasNext()) { - byte[] keyInSource = keysFromSource.next(); - // Fetch the value given the keys. - Optional valFromKey = detailsSrc.get(keyInSource); - - // Add to total size given size of the value - totalSize += valFromKey.map(rawDetails -> rawDetails.length).orElse(0); - } - - // Flushes both details and storage. - detailsSrc.commit(); - storageSrc.commit(); - - return totalSize; - } - - public void syncLargeStorage() { - - Iterator keysFromSource = detailsSrc.keys(); - while (keysFromSource.hasNext()) { - byte[] keyInSource = keysFromSource.next(); - - // Fetch the value given the keys. - Optional rawDetails = detailsSrc.get(keyInSource); - - // If it is null, just continue - if (!rawDetails.isPresent()) { - continue; - } - - // Decode the details. - AionContractDetailsImpl detailsImpl = new AionContractDetailsImpl(storageDSPrune, graphSrc); - detailsImpl.decode(rawDetails.get(), true); - // We can safely get as we checked if it is present. - - // ContractDetails details = entry.getValue(); - detailsImpl.syncStorage(); - } - } - public JournalPruneDataSource getStorageDSPrune() { return storageDSPrune; } diff --git a/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainAccountStateBenchmark.java b/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainAccountStateBenchmark.java index 95ca7169f0..329e28ae72 100644 --- a/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainAccountStateBenchmark.java +++ b/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainAccountStateBenchmark.java @@ -338,7 +338,7 @@ public void testExpandContractsStorage() throws InterruptedException { AionContractDetailsImpl acdi = new AionContractDetailsImpl( bc.getRepository().getContractDetails(contractAddress).getEncoded()); - assertFalse(acdi.externalStorage); + assertFalse(acdi.isExternalStorage()); // around 350 tx to letting the contract storage from memory switch to the external // storage. @@ -349,7 +349,7 @@ public void testExpandContractsStorage() throws InterruptedException { acdi = new AionContractDetailsImpl( bc.getRepository().getContractDetails(contractAddress).getEncoded()); - assertTrue(acdi.externalStorage); + assertTrue(acdi.isExternalStorage()); } catch (Exception e) { e.printStackTrace(); diff --git a/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainTestUtils.java b/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainTestUtils.java index b3e0abdc89..416c460bb9 100644 --- a/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainTestUtils.java +++ b/modAionImpl/test/org/aion/zero/impl/blockchain/BlockchainTestUtils.java @@ -212,6 +212,10 @@ public static void generateRandomChainWithoutTransactions(StandaloneBlockchain c } } + public static Pair addMiningBlock(StandaloneBlockchain chain, Block parent, List txs) { + return addMiningBlock(chain, parent, txs, System.currentTimeMillis(), new byte[0]); + } + private static Pair addMiningBlock(StandaloneBlockchain chain, Block parent, List txs, long time, byte[] extraData) { AionBlock block = chain.createNewMiningBlockInternal(parent, txs, true, time / TEN_THOUSAND_MS).block; @@ -305,7 +309,7 @@ public static void generateRandomUnityChain(StandaloneBlockchain chain, TestReso txs.add(registerStaker(resourceProvider.factoryForVersion1, key, MIN_SELF_STAKE, repo.getNonce(new AionAddress(key.getAddress())), contract)); } } else { - txs = generateTransactions(txCount, accounts, repo); + txs = txCount > 0 ? generateTransactions(txCount, accounts, repo) : Collections.emptyList(); } // get back to current root repo.syncToRoot(originalRoot); @@ -722,6 +726,11 @@ private static AionTransaction finalizeTransfer(IAvmResourceFactory avmFactory, null); } + public static Pair addStakingBlock(StandaloneBlockchain chain, Block parent, List txs, ECKey key) { + byte[] parentSeed = chain.forkUtility.isUnityForkBlock(parent.getNumber()) ? StakingBlockHeader.GENESIS_SEED : ((StakingBlock) chain.getBlockByHash(parent.getParentHash())).getSeed(); + return addStakingBlock(chain, parentSeed, txs, System.currentTimeMillis(), key); + } + private static Pair addStakingBlock( StandaloneBlockchain chain, byte[] parentSeed, @@ -811,4 +820,53 @@ public static Block generateNewBlock( block.updateHeader(newBlockHeader); return block; } + + public static AionTransaction deployLargeStorageContractTransaction(IAvmResourceFactory avmFactory, ECKey owner, BigInteger nonce) { + byte[] jar = avmFactory.newContractFactory().getDeploymentBytes(AvmContract.LARGE_STORAGE); + return AionTransaction.create( + owner, + nonce.toByteArray(), + null, + new byte[0], + jar, + LIMIT_DEPLOY, + NRG_PRICE, + TransactionTypes.AVM_CREATE_CODE, + null); + } + + public static AionTransaction putToLargeStorageTransaction(IAvmResourceFactory avmFactory, ECKey caller, byte[] key, byte[] value, BigInteger nonce, AionAddress contract) { + byte[] callBytes = avmFactory.newStreamingEncoder() + .encodeOneString("putStorage") + .encodeOneByteArray(key) + .encodeOneByteArray(value) + .getEncoding(); + return AionTransaction.create( + caller, + nonce.toByteArray(), + contract, + BigInteger.ZERO.toByteArray(), + callBytes, + LIMIT_CALL, + NRG_PRICE, + TransactionTypes.DEFAULT, + null); + } + + public static AionTransaction getFromLargeStorageTransaction(IAvmResourceFactory avmFactory, ECKey caller, byte[] key, BigInteger nonce, AionAddress contract) { + byte[] callBytes = avmFactory.newStreamingEncoder() + .encodeOneString("getStorage") + .encodeOneByteArray(key) + .getEncoding(); + return AionTransaction.create( + caller, + nonce.toByteArray(), + contract, + BigInteger.ZERO.toByteArray(), + callBytes, + LIMIT_CALL, + NRG_PRICE, + TransactionTypes.DEFAULT, + null); + } } diff --git a/modAionImpl/test/org/aion/zero/impl/db/AionContractDetailsTest.java b/modAionImpl/test/org/aion/zero/impl/db/AionContractDetailsTest.java index f213e61d99..1c8c7ae876 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/AionContractDetailsTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/AionContractDetailsTest.java @@ -5,6 +5,7 @@ import static org.aion.zero.impl.db.DatabaseUtils.connectAndOpen; import static org.aion.util.bytes.ByteUtil.EMPTY_BYTE_ARRAY; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -12,6 +13,7 @@ import java.util.Map; import java.util.Properties; import org.aion.db.impl.ByteArrayKeyValueDatabase; +import org.aion.db.impl.ByteArrayKeyValueStore; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; import org.aion.db.store.JournalPruneDataSource; @@ -34,9 +36,6 @@ public class AionContractDetailsTest { private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); - private static final int IN_MEMORY_STORAGE_LIMIT = - 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); - protected RepositoryConfig repoConfig = new RepositoryConfig() { @Override @@ -57,15 +56,6 @@ public Properties getDatabaseConfig(String db_name) { } }; - private static AionContractDetailsImpl deserialize( - byte[] rlp, ByteArrayKeyValueDatabase externalStorage) { - AionContractDetailsImpl result = new AionContractDetailsImpl(); - result.setExternalStorageDataSource(externalStorage); - result.decode(rlp); - - return result; - } - @Test public void test_1() throws Exception { @@ -77,11 +67,7 @@ public void test_1() throws Exception { byte[] key_2 = ByteUtil.hexStringToBytes("222222"); byte[] val_2 = ByteUtil.hexStringToBytes("bbbbbb"); - AionContractDetailsImpl contractDetails = - new AionContractDetailsImpl( - -1, // CfgAion.inst().getDb().getPrune(), - 1000000 // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() - ); + AionContractDetailsImpl contractDetails = new AionContractDetailsImpl(); contractDetails.setCode(code); contractDetails.setVmType(InternalVmType.FVM); contractDetails.put( @@ -267,17 +253,16 @@ public void testExternalStorageSerialization() { Map elements = new HashMap<>(); AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); - ByteArrayKeyValueDatabase externalStorage = repository.getDetailsDatabase(); + ByteArrayKeyValueStore externalStorage = repository.detailsDS.getStorageDSPrune(); + ByteArrayKeyValueDatabase graphDatabase = repository.graphDatabase; - AionContractDetailsImpl original = new AionContractDetailsImpl(0, 1000000); - - original.setExternalStorageDataSource(externalStorage); + AionContractDetailsImpl original = new AionContractDetailsImpl(externalStorage, graphDatabase); original.setAddress(address); + original.initializeExternalStorageTrieForTest(); original.setCode(code); original.setVmType(InternalVmType.FVM); - original.externalStorage = true; - for (int i = 0; i < IN_MEMORY_STORAGE_LIMIT / 64 + 10; i++) { + for (int i = 0; i < AionContractDetailsImpl.detailsInMemoryStorageLimit / 64 + 10; i++) { DataWord key = new DataWord(RandomUtils.nextBytes(16)); DataWord value = new DataWord(RandomUtils.nextBytes(16)); @@ -289,11 +274,10 @@ public void testExternalStorageSerialization() { byte[] rlp = original.getEncoded(); - AionContractDetailsImpl deserialized = new AionContractDetailsImpl(); - deserialized.setExternalStorageDataSource(externalStorage); + AionContractDetailsImpl deserialized = new AionContractDetailsImpl(externalStorage, graphDatabase); deserialized.decode(rlp); - assertEquals(deserialized.externalStorage, true); + assertTrue(deserialized.isExternalStorage()); assertTrue(address.equals(deserialized.getAddress())); byte[] codeHash = h256(code); assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(deserialized.getCode(codeHash))); @@ -316,8 +300,7 @@ public void testContractStorageSwitch() { byte[] code = RandomUtils.nextBytes(512); Map elements = new HashMap<>(); - int memstoragelimit = 512; - AionContractDetailsImpl original = new AionContractDetailsImpl(0, memstoragelimit); + AionContractDetailsImpl.detailsInMemoryStorageLimit = 512; // getting storage specific properties Properties sharedProps; @@ -327,7 +310,7 @@ public void testContractStorageSwitch() { sharedProps.setProperty(DatabaseFactory.Props.DB_NAME, "storage"); ByteArrayKeyValueDatabase storagedb = connectAndOpen(sharedProps, LOG); JournalPruneDataSource jpd = new JournalPruneDataSource(storagedb, LOG); - original.setDataSource(jpd); + AionContractDetailsImpl original = new AionContractDetailsImpl(jpd, null); original.setAddress(address); original.setCode(code); original.setVmType(InternalVmType.FVM); @@ -343,7 +326,7 @@ public void testContractStorageSwitch() { original.decode(original.getEncoded()); original.syncStorage(); - assertTrue(!original.externalStorage); + assertFalse(original.isExternalStorage()); // transfer to external storage since 3rd insert DataWord key3rd = new DataWord(RandomUtils.nextBytes(16)); @@ -353,15 +336,14 @@ public void testContractStorageSwitch() { original.decode(original.getEncoded()); original.syncStorage(); - assertTrue(original.externalStorage); + assertTrue(original.isExternalStorage()); byte[] rlp = original.getEncoded(); - AionContractDetailsImpl deserialized = new AionContractDetailsImpl(0, memstoragelimit); - deserialized.setDataSource(jpd); + AionContractDetailsImpl deserialized = new AionContractDetailsImpl(jpd, null); deserialized.decode(rlp); - assertTrue(deserialized.externalStorage); + assertTrue(deserialized.isExternalStorage()); assertEquals(address, deserialized.getAddress()); byte[] codeHash = h256(code); assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(deserialized.getCode(codeHash))); @@ -376,6 +358,9 @@ public void testContractStorageSwitch() { deserialized.delete(deletedKey.toWrapper()); deserialized.delete(new DataWord(RandomUtils.nextBytes(16)).toWrapper()); + + // reset static variable to original value + AionContractDetailsImpl.detailsInMemoryStorageLimit = 64 * 1024; } @Test @@ -385,15 +370,15 @@ public void testExternalStorageTransition() { Map elements = new HashMap<>(); AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); - ByteArrayKeyValueDatabase externalStorage = repository.getDetailsDatabase(); + ByteArrayKeyValueStore externalStorage = repository.detailsDS.getStorageDSPrune(); + ByteArrayKeyValueDatabase graphDatabase = repository.graphDatabase; - AionContractDetailsImpl original = new AionContractDetailsImpl(0, 1000000); - original.setExternalStorageDataSource(externalStorage); + AionContractDetailsImpl original = new AionContractDetailsImpl(externalStorage, graphDatabase); original.setAddress(address); original.setCode(code); original.setVmType(InternalVmType.FVM); - for (int i = 0; i < IN_MEMORY_STORAGE_LIMIT / 64 + 10; i++) { + for (int i = 0; i < AionContractDetailsImpl.detailsInMemoryStorageLimit / 64 + 10; i++) { DataWord key = new DataWord(RandomUtils.nextBytes(16)); DataWord value = new DataWord(RandomUtils.nextBytes(16)); @@ -401,10 +386,14 @@ public void testExternalStorageTransition() { original.put(key.toWrapper(), wrapValueForPut(value)); } + /* NOTE: This operation does not actually transition the storage to the external database. + The transition occurs only during decoding. */ original.syncStorage(); - assertTrue(!externalStorage.isEmpty()); + assertTrue(externalStorage.isEmpty()); - AionContractDetailsImpl deserialized = deserialize(original.getEncoded(), externalStorage); + AionContractDetailsImpl deserialized = new AionContractDetailsImpl(externalStorage, graphDatabase); + deserialized.decode(original.getEncoded()); + assertTrue(deserialized.isExternalStorage()); // adds keys for in-memory storage limit overflow for (int i = 0; i < 10; i++) { @@ -419,12 +408,13 @@ public void testExternalStorageTransition() { deserialized.syncStorage(); assertTrue(!externalStorage.isEmpty()); - deserialized = deserialize(deserialized.getEncoded(), externalStorage); + AionContractDetailsImpl deserialized2 = new AionContractDetailsImpl(externalStorage, graphDatabase); + deserialized2.decode(deserialized.getEncoded()); for (DataWord key : elements.keySet()) { assertEquals( elements.get(key).toWrapper(), - wrapValueFromGet(deserialized.get(key.toWrapper()))); + wrapValueFromGet(deserialized2.get(key.toWrapper()))); } } @@ -470,10 +460,10 @@ public void testEncodingCorrectSize() throws Exception { byte[] code = RandomUtils.nextBytes(512); AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); - ByteArrayKeyValueDatabase externalStorage = repository.getDetailsDatabase(); + ByteArrayKeyValueStore externalStorage = repository.detailsDS.getStorageDSPrune(); + ByteArrayKeyValueDatabase graphDatabase = repository.graphDatabase; - AionContractDetailsImpl details = new AionContractDetailsImpl(0, 1000000); - details.setExternalStorageDataSource(externalStorage); + AionContractDetailsImpl details = new AionContractDetailsImpl(externalStorage, graphDatabase); details.setAddress(address); details.setCode(code); details.setVmType(InternalVmType.FVM); diff --git a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java index f8a5d989ec..dbf6f2b8e4 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java @@ -157,7 +157,7 @@ public void testAccountStateUpdateStorageRowFlush() { assertThat(serializedDetails.isPresent()).isEqualTo(true); - AionContractDetailsImpl details = new AionContractDetailsImpl(0, 1000000); + AionContractDetailsImpl details = new AionContractDetailsImpl(); details.decode(serializedDetails.get()); assertThat(details.get(new DataWord(key).toWrapper())) .isEqualTo(new DataWord(value).toWrapper()); diff --git a/modAionImpl/test/org/aion/zero/impl/db/DetailsDataStoreIntegTest.java b/modAionImpl/test/org/aion/zero/impl/db/DetailsDataStoreIntegTest.java new file mode 100644 index 0000000000..e953706295 --- /dev/null +++ b/modAionImpl/test/org/aion/zero/impl/db/DetailsDataStoreIntegTest.java @@ -0,0 +1,460 @@ +package org.aion.zero.impl.db; + +import static com.google.common.truth.Truth.assertThat; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.addMiningBlock; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.addStakingBlock; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.deployLargeStorageContractTransaction; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.generateAccounts; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.generateNextMiningBlock; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.generateNextStakingBlock; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.generateRandomUnityChain; +import static org.aion.zero.impl.blockchain.BlockchainTestUtils.putToLargeStorageTransaction; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.aion.base.AionTransaction; +import org.aion.base.TransactionTypeRule; +import org.aion.base.TxUtil; +import org.aion.crypto.ECKey; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.aion.log.LogLevel; +import org.aion.mcf.blockchain.Block; +import org.aion.types.AionAddress; +import org.aion.zero.impl.blockchain.AionBlockchainImpl; +import org.aion.zero.impl.blockchain.StandaloneBlockchain; +import org.aion.zero.impl.core.ImportResult; +import org.aion.zero.impl.vm.AvmPathManager; +import org.aion.zero.impl.vm.AvmTestConfig; +import org.aion.zero.impl.vm.TestResourceProvider; +import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Unit and integration tests for {@link DetailsDataStore}. + * + * @author Alexandra Roatis + */ +public class DetailsDataStoreIntegTest { + private static final long unityForkBlock = 2; + private TestResourceProvider resourceProvider; + + @Before + public void setup() + throws ClassNotFoundException, InstantiationException, IllegalAccessException, + IOException { + // reduce default logging levels + AionLoggerFactory.initAll(); + + resourceProvider = TestResourceProvider.initializeAndCreateNewProvider(AvmPathManager.getPathOfProjectRootDirectory()); + + // enable both AVMs without overlap + AvmTestConfig.supportBothAvmVersions(0, unityForkBlock, 0); + + TransactionTypeRule.allowAVMContractTransaction(); + } + + @After + public void tearDown() { + AvmTestConfig.clearConfigurations(); + } + + /** + * Creates a contract with large storage. Ensures that the trie transitions from the details + * database to the storage database after reaching the transition size. + */ + @Test + public void largeStorageTransition() { + int txPerBlock = 60; + int numberOfStorageBlocks = 2; // storage transitions after the second call block is added + + // setup accounts + List accounts = generateAccounts(3); + ECKey stakingRegistryOwner = accounts.get(0); + ECKey staker = accounts.get(1); + ECKey account = accounts.get(2); // used to deploy and call the large storage contract + BigInteger nonce = BigInteger.ZERO; + + // setup two identical blockchains + StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder(); + StandaloneBlockchain chain = + builder.withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .withAvmEnabled() + .build() + .bc; + chain.forkUtility.enableUnityFork(unityForkBlock); + + // populate the chain for unity + generateRandomUnityChain(chain, resourceProvider, 3, 1, List.of(staker), stakingRegistryOwner, 0); + + // tracks all the deployed storage contracts + List txs = new ArrayList<>(); + + // deploy the storage contract + AionTransaction tx = + deployLargeStorageContractTransaction( + resourceProvider.factoryForVersion2, account, nonce); + nonce = nonce.add(BigInteger.ONE); + addMiningBlock(chain, chain.getBestBlock(), List.of(tx)); + + // save the contract address + AionAddress contract = TxUtil.calculateContractAddress(tx); + System.out.println("Contract: " + contract); + + boolean stakingIsNext = true; + for (int i = 0; i < numberOfStorageBlocks; i++) { + // verify that the storage is empty + assertThat(chain.getRepository().storageDatabase.isEmpty()).isTrue(); + + // call contracts to increase storage + for (int j = 0; j < txPerBlock; j++) { + txs.add( + putToLargeStorageTransaction( + resourceProvider.factoryForVersion2, + account, + RandomUtils.nextBytes(32), + RandomUtils.nextBytes(32), + nonce, + contract)); + nonce = nonce.add(BigInteger.ONE); + } + + if (stakingIsNext) { + addStakingBlock(chain, chain.getBestBlock(), txs, staker); + } else { + addMiningBlock(chain, chain.getBestBlock(), txs); + } + stakingIsNext = !stakingIsNext; + } + + // verify that the storage has transitioned + assertThat(chain.getRepository().storageDatabase.isEmpty()).isFalse(); + } + + /** + * Creates a contract with large storage. Imports 20 blocks expanding the storage and prints + * times and database size with and without storage transition. + */ + @Ignore + @Test + public void largeStorageTransitionBenchmark() throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException { + // setup account used to deploy and call the large storage contract + List accounts = generateAccounts(1); + ECKey account = accounts.get(0); + + // setup AVM + AvmTestConfig.clearConfigurations(); // clear setting from @Before + TransactionTypeRule.allowAVMContractTransaction(); + resourceProvider = TestResourceProvider.initializeAndCreateNewProvider(AvmPathManager.getPathOfProjectRootDirectory()); + AvmTestConfig.supportBothAvmVersions(0, 1, 0); // enable both AVMs without overlap + + // setup 3 identical blockchains + StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder(); + StandaloneBlockchain chain = builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).withAvmEnabled().build().bc; + StandaloneBlockchain chainWithTransition = builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).withAvmEnabled().build().bc; + StandaloneBlockchain chainWithoutTransition = builder.withValidatorConfiguration("simple").withDefaultAccounts(accounts).withAvmEnabled().build().bc; + + // disabling Unity for this test + chain.forkUtility.disableUnityFork(); + chainWithTransition.forkUtility.disableUnityFork(); + chainWithoutTransition.forkUtility.disableUnityFork(); + + assertThat(chain).isNotEqualTo(chainWithTransition); + assertThat(chain.genesis).isEqualTo(chainWithTransition.genesis); + assertThat(chain).isNotEqualTo(chainWithoutTransition); + assertThat(chain.genesis).isEqualTo(chainWithoutTransition.genesis); + assertThat(chainWithoutTransition).isNotEqualTo(chainWithTransition); + assertThat(chainWithoutTransition.genesis).isEqualTo(chainWithTransition.genesis); + + // deploy the storage contract + BigInteger nonce = BigInteger.ZERO; + AionTransaction tx = deployLargeStorageContractTransaction(resourceProvider.factoryForVersion2, account, nonce); + nonce = nonce.add(BigInteger.ONE); + addMiningBlock(chain, chain.getBestBlock(), List.of(tx)); + + // save the contract address + AionAddress contract = TxUtil.calculateContractAddress(tx); + System.out.println("Contract: " + contract); + + List txs = new ArrayList<>(); + int insertTxPerBlock = 9, updateTxPerBlock = 5, deleteTxPerBlock = 4; + int numberOfStorageBlocks = 2*6; // results in 3*6 blocks with storage in details, 3*6 after transition + LinkedList keys = new LinkedList<>(); + + // populate chain + for (int i = 0; i < numberOfStorageBlocks; i++) { + // call contract to increase storage + for (int j = 0; j < insertTxPerBlock; j++) { + byte[] key = RandomUtils.nextBytes(32 - i); + txs.add( + putToLargeStorageTransaction( + resourceProvider.factoryForVersion2, + account, + key, + RandomUtils.nextBytes(32 - i), + nonce, + contract)); + nonce = nonce.add(BigInteger.ONE); + keys.addLast(key); + } + addMiningBlock(chain, chain.getBestBlock(), txs); + + // call contract to update storage + for (int j = 0; j < updateTxPerBlock; j++) { + byte[] key = keys.removeFirst(); + txs.add( + putToLargeStorageTransaction( + resourceProvider.factoryForVersion2, + account, + key, + RandomUtils.nextBytes(32 - i - 1), + nonce, + contract)); + nonce = nonce.add(BigInteger.ONE); + keys.addLast(key); + } + addMiningBlock(chain, chain.getBestBlock(), txs); + + // call contract to delete storage + for (int j = 0; j < deleteTxPerBlock; j++) { + txs.add( + putToLargeStorageTransaction( + resourceProvider.factoryForVersion2, + account, + keys.removeFirst(), + null, + nonce, + contract)); + nonce = nonce.add(BigInteger.ONE); + } + addMiningBlock(chain, chain.getBestBlock(), txs); + } + + // import the contract block (not part of the benchmark) + Block block = chain.getBlockByNumber(1); + ImportResult importResult = chainWithoutTransition.tryToConnect(block); + assertThat(importResult).isEqualTo(ImportResult.IMPORTED_BEST); + importResult = chainWithTransition.tryToConnect(block); + assertThat(importResult).isEqualTo(ImportResult.IMPORTED_BEST); + + long start, duration, storage, details, size; + long totalTimeWithTransition = 0, + totalDetailsSizeWithTransition = 0, + totalStorageSizeWithTransition = 0, + totalSizeWithTransition = 0, + totalTimeWithoutTransition = 0, + totalDetailsSizeWithoutTransition = 0, + totalStorageSizeWithoutTransition = 0, + totalSizeWithoutTransition = 0; + + int blocks = (int) chain.getBestBlock().getNumber(); + long results[][] = new long[blocks + 1][8]; + boolean isEmptyStorage = true, isSaved = false; + + // generate data by importing each block to the chain with/without transition + for (int i = 2; i <= blocks; i++) { + block = chain.getBlockByNumber(i); + + // importing without transition first + // In case there is any bias due to caching, it will disfavour this option. + // Even with any potential nevative bias, the option seems to have better performance. + AionContractDetailsImpl.detailsInMemoryStorageLimit = 0; + + start = System.nanoTime(); + importResult = chainWithoutTransition.tryToConnect(block); + duration = System.nanoTime() - start; + assertThat(importResult).isEqualTo(ImportResult.IMPORTED_BEST); + details = chainWithoutTransition.getRepository().detailsDatabase.approximateSize(); + storage = chainWithoutTransition.getRepository().storageDatabase.approximateSize(); + size = details + storage; + + results[i][4] = duration; + results[i][5] = details; + results[i][6] = storage; + results[i][7] = size; + + totalTimeWithoutTransition += duration; + totalDetailsSizeWithoutTransition += details; + totalStorageSizeWithoutTransition += storage; + totalSizeWithoutTransition += size; + + // importing with transition second + AionContractDetailsImpl.detailsInMemoryStorageLimit = 65536; + + start = System.nanoTime(); + importResult = chainWithTransition.tryToConnect(block); + duration = System.nanoTime() - start; + assertThat(importResult).isEqualTo(ImportResult.IMPORTED_BEST); + details = chainWithTransition.getRepository().detailsDatabase.approximateSize(); + storage = chainWithTransition.getRepository().storageDatabase.approximateSize(); + size = details + storage; + + results[i][0] = duration; + results[i][1] = details; + results[i][2] = storage; + results[i][3] = size; + + totalTimeWithTransition += duration; + totalDetailsSizeWithTransition += details; + totalStorageSizeWithTransition += storage; + totalSizeWithTransition += size; + + if (storage != 0 && isEmptyStorage) { + isEmptyStorage = false; + } + + if (!isEmptyStorage && !isSaved) { + results[1][0] = totalTimeWithTransition; + results[1][1] = totalDetailsSizeWithTransition; + results[1][2] = totalStorageSizeWithTransition; + results[1][3] = totalSizeWithTransition; + results[1][4] = totalTimeWithoutTransition; + results[1][5] = totalDetailsSizeWithoutTransition; + results[1][6] = totalStorageSizeWithoutTransition; + results[1][7] = totalSizeWithoutTransition; + isSaved = true; + } + } + + // Save totals + results[0][0] = totalTimeWithTransition; + results[0][1] = totalDetailsSizeWithTransition; + results[0][2] = totalStorageSizeWithTransition; + results[0][3] = totalSizeWithTransition; + results[0][4] = totalTimeWithoutTransition; + results[0][5] = totalDetailsSizeWithoutTransition; + results[0][6] = totalStorageSizeWithoutTransition; + results[0][7] = totalSizeWithoutTransition; + + print(results); + } + + private static void print(long results[][]) { + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + System.out.printf( + "| %8s | %10s | %53s | %53s | %25s |\n", + " ", " ", "With Storage Transition", "Without Storage Transition", "Difference"); + System.out.printf( + "| %8s | %10s | %11s | %11s | %11s | %11s | %11s | %11s | %11s | %11s | %11s | %11s |\n", + " ", "Type", + "Time (ns)", + "Detais (B)", + "Storage (B)", + "Size (B)", + "Time (ns)", + "Detais (B)", + "Storage (B)", + "Size (B)", + "Time (ns)", + "Size (B)"); + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + System.out.printf( + "| %21s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + "Total", + results[0][0], + results[0][1], + results[0][2], + results[0][3], + results[0][4], + results[0][5], + results[0][6], + results[0][7], + (results[0][0] - results[0][4]), + (results[0][3] - results[0][7])); + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + System.out.printf( + "| %21s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + "Read In-line Total", + results[1][0], + results[1][1], + results[1][2], + results[1][3], + results[1][4], + results[1][5], + results[1][6], + results[1][7], + (results[1][0] - results[1][4]), + (results[1][3] - results[1][7])); + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + // aggregate data for each type of update + long[][] extra = new long[3][8]; + for (int i = 2; (i < results.length) && (results[i][2] == 0); i++) { + int k = i % 3; + for (int j = 0; j < 8; j++) { + extra[k][j] += results[i][j]; + } + } + // inserts are at block nb % 3 = 2 + System.out.printf( + "| %21s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + "Insert In-line Total", + extra[2][0], + extra[2][1], + extra[2][2], + extra[2][3], + extra[2][4], + extra[2][5], + extra[2][6], + extra[2][7], + (extra[2][0] - extra[2][4]), + (extra[2][3] - extra[2][7])); + // updates are at block nb % 3 = 0 + System.out.printf( + "| %21s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + "Update In-line Total", + extra[0][0], + extra[0][1], + extra[0][2], + extra[0][3], + extra[0][4], + extra[0][5], + extra[0][6], + extra[0][7], + (extra[0][0] - extra[0][4]), + (extra[0][3] - extra[0][7])); + // deletes are at block nb % 3 = 1 + System.out.printf( + "| %21s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + "Delete In-line Total", + extra[1][0], + extra[1][1], + extra[1][2], + extra[1][3], + extra[1][4], + extra[1][5], + extra[1][6], + extra[1][7], + (extra[1][0] - extra[1][4]), + (extra[1][3] - extra[1][7])); + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + for (int i = 2; i < results.length; i++) { + System.out.printf( + "| Block %2d | %10s | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d | %11d |\n", + i, + (i % 3 == 0 ? "5 tx UPD" : (i % 3 == 1 ? "4 tx DEL" : "9 tx INS")), + results[i][0], + results[i][1], + results[i][2], + results[i][3], + results[i][4], + results[i][5], + results[i][6], + results[i][7], + (results[i][0] - results[i][4]), + (results[i][3] - results[i][7])); + } + System.out.printf("---------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + } +} diff --git a/modAvmStub/src/org/aion/avm/stub/IContractFactory.java b/modAvmStub/src/org/aion/avm/stub/IContractFactory.java index 7bfcd6b3ab..cfa16a56b5 100644 --- a/modAvmStub/src/org/aion/avm/stub/IContractFactory.java +++ b/modAvmStub/src/org/aion/avm/stub/IContractFactory.java @@ -9,7 +9,7 @@ * just that production code has no use for it (at the moment anyway). */ public interface IContractFactory { - public enum AvmContract { HELLO_WORLD, GENERIC_CONTRACT, INTERNAL_TRANSACTION, LOG_TARGET, STATEFULNESS, TRANSACTION_HASH, META_TRANSACTION_PROXY, UNITY_STAKER_REGISTRY } + public enum AvmContract { HELLO_WORLD, GENERIC_CONTRACT, INTERNAL_TRANSACTION, LOG_TARGET, STATEFULNESS, TRANSACTION_HASH, META_TRANSACTION_PROXY, UNITY_STAKER_REGISTRY, LARGE_STORAGE } /** * Returns the bytes of the specified contract in the specific encoding that the avm expects. diff --git a/modAvmVersion2/src/org/aion/avm/version2/ContractFactory.java b/modAvmVersion2/src/org/aion/avm/version2/ContractFactory.java index 7328e27a8f..7757dd73e9 100644 --- a/modAvmVersion2/src/org/aion/avm/version2/ContractFactory.java +++ b/modAvmVersion2/src/org/aion/avm/version2/ContractFactory.java @@ -7,6 +7,7 @@ import org.aion.avm.version2.contracts.GenericContract; import org.aion.avm.version2.contracts.HelloWorld; import org.aion.avm.version2.contracts.InternalTransaction; +import org.aion.avm.version2.contracts.LargeStorage; import org.aion.avm.version2.contracts.LogTarget; import org.aion.avm.version2.contracts.MetaTransactionProxy; import org.aion.avm.version2.contracts.Statefulness; @@ -35,6 +36,8 @@ public byte[] getDeploymentBytes(AvmContract contract) { return new CodeAndArguments(getOptimizedDappBytes(TransactionHash.class), new byte[0]).encodeToBytes(); case META_TRANSACTION_PROXY: return new CodeAndArguments(getOptimizedDappBytes(MetaTransactionProxy.class), new byte[0]).encodeToBytes(); + case LARGE_STORAGE: + return new CodeAndArguments(getOptimizedDappBytes(LargeStorage.class), new byte[0]).encodeToBytes(); default : throw new IllegalStateException("The following contract is not supported by version 2 of the avm: " + contract); } } diff --git a/modAvmVersion2/src/org/aion/avm/version2/contracts/LargeStorage.java b/modAvmVersion2/src/org/aion/avm/version2/contracts/LargeStorage.java new file mode 100644 index 0000000000..695ce8fdd6 --- /dev/null +++ b/modAvmVersion2/src/org/aion/avm/version2/contracts/LargeStorage.java @@ -0,0 +1,39 @@ +package org.aion.avm.version2.contracts; + +import avm.Blockchain; +import java.math.BigInteger; +import org.aion.avm.tooling.abi.Callable; + +/** + * This contract provides methods for adding data to storage. It counts and logs the number of calls + * made to add/update stored data. It also allows querying the stored data. + */ +public class LargeStorage { + + /** + * This counter is not intended as a storage key counter. Its purpose is to make changes to the + * object graph as well as the storage. + */ + private static BigInteger countPutCalls = BigInteger.ZERO; + + @Callable + public static void putStorage(byte[] key, byte[] value) { + Blockchain.putStorage(convertToFittingKey(key), value); + countPutCalls = countPutCalls.add(BigInteger.ONE); + Blockchain.log(countPutCalls.toByteArray()); + } + + @Callable + public static String getStorage(byte[] key) { + byte[] payload = Blockchain.getStorage(convertToFittingKey(key)); + return (null != payload) ? new String(payload) : null; + } + + private static byte[] convertToFittingKey(byte[] raw) { + // The key needs to be 32-bytes so either truncate or 0x0-pad the bytes from the string. + byte[] key = new byte[32]; + int length = StrictMath.min(key.length, raw.length); + System.arraycopy(raw, 0, key, 0, length); + return key; + } +} diff --git a/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java b/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java index caf34cca8a..696173f84c 100644 --- a/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java +++ b/modDbImpl/src/org/aion/db/impl/mockdb/MockDB.java @@ -6,6 +6,7 @@ import java.util.Iterator; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.aion.db.impl.AbstractDB; import org.aion.db.impl.PersistenceMethod; import org.aion.util.types.ByteArrayWrapper; @@ -70,7 +71,7 @@ public boolean isCreatedOnDisk() { @Override public long approximateSize() { check(); - return -1L; + return 32L * kv.size() + kv.values().stream().map(k -> (long) k.length).reduce(0L, Long::sum); } @Override diff --git a/modMcf/src/org/aion/mcf/db/ContractDetails.java b/modMcf/src/org/aion/mcf/db/ContractDetails.java index 2db2a789ae..5d9c3d0757 100644 --- a/modMcf/src/org/aion/mcf/db/ContractDetails.java +++ b/modMcf/src/org/aion/mcf/db/ContractDetails.java @@ -92,12 +92,8 @@ public interface ContractDetails { */ byte[] getStorageHash(); - /** - * Sets the dirty value to dirty. - * - * @param dirty The dirty value. - */ - void setDirty(boolean dirty); + /** Marks the contract as dirty (i.e. to be written). */ + void markAsDirty(); /** Marks the contract as deleted. */ void delete();