Merge Besu main

CHANGELOG.md

@@ -1,8 +1,9 @@
 # Changelog
 
-## 23.10.4
+## 24.1.0-SNAPSHOT
 
 ### Breaking Changes
+- New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343)
 
 ### Deprecations
 - Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230)
@@ -13,16 +14,13 @@
 - Set Ethereum Classic mainnet activation block for Spiral network upgrade [#6267](https://github.com/hyperledger/besu/pull/6267)
 - Add custom genesis file name to config overview if specified [#6297](https://github.com/hyperledger/besu/pull/6297)
 - Update Gradle plugins and replace unmaintained License Gradle Plugin with the actively maintained Gradle License Report [#6275](https://github.com/hyperledger/besu/pull/6275)
+- Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328)
 
 ### Bug fixes
-
+- INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344)
 
 ## 23.10.3
 
-### Breaking Changes
-
-### Deprecations
-
 ### Additions and Improvements
 - Implement debug_traceCall [#5885](https://github.com/hyperledger/besu/pull/5885)
 - Transactions that takes too long to evaluate, during block creation, are dropped from the txpool [#6163](https://github.com/hyperledger/besu/pull/6163)
@@ -34,11 +32,18 @@
 - Update OpenJ9 Docker image to latest version [#6226](https://github.com/hyperledger/besu/pull/6226)
 - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212)
 - Add `rocksdb usage` to the `storage` subcommand to allow users and dev to check columns families usage [#6185](https://github.com/hyperledger/besu/pull/6185)
+- Ethereum Classic Spiral network upgrade [#6078](https://github.com/hyperledger/besu/pull/6078)
+- Fix self destruct collision [#6205](https://github.com/hyperledger/besu/pull/6205)
+- Mark deleted storage on cleared [#6305](https://github.com/hyperledger/besu/pull/6305)
 
 ### Bug fixes
 - Fix Docker image name clash between Besu and evmtool [#6194](https://github.com/hyperledger/besu/pull/6194)
 - Fix `logIndex` in `eth_getTransactionReceipt` JSON RPC method [#6206](https://github.com/hyperledger/besu/pull/6206)
 
+### Download Links
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.zip / sha256 da7ef8a6ceb88d3e327cacddcdb32218d1750b464c14165a74068f6dc6e0871a
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.10.3/besu-23.10.3.tar.gz / sha256 73c834cf32c7bbe255d7d8cc7ca5d1eb0df8430b9114935c8dcf3a675b2acbc2
+
 ## 23.10.2
 
 ### Breaking Changes

README.md

@@ -47,7 +47,7 @@ Instructions for how to get started with developing on the Besu codebase. Please
 * [Checking Out and Building](https://wiki.hyperledger.org/display/BESU/Building+from+source)
 * [Running Developer Builds](https://wiki.hyperledger.org/display/BESU/Building+from+source#running-developer-builds)
 * [Code Coverage](https://wiki.hyperledger.org/display/BESU/Code+coverage)
-* [Logging](https://wiki.hyperledger.org/display/BESU/Logging) or the [Documentation's Logging section](https://besu.hyperledger.org/en/stable/HowTo/Monitor/Logging/)
+* [Logging](https://wiki.hyperledger.org/display/BESU/Logging) or the [Documentation's Logging section](https://besu.hyperledger.org/public-networks/how-to/monitor/logging)
 
 
 ## Release Notes

acceptance-tests/dsl/build.gradle

@@ -47,4 +47,5 @@ dependencies {
   implementation 'org.web3j:crypto'
 
   implementation 'org.testcontainers:testcontainers'
+  implementation 'org.junit.jupiter:junit-jupiter'
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBase.java

@@ -49,7 +49,6 @@
 import org.hyperledger.besu.tests.acceptance.dsl.transaction.web3.Web3Transactions;
 
 import java.io.BufferedReader;
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.lang.ProcessBuilder.Redirect;
@@ -58,14 +57,15 @@
 import java.util.concurrent.Executors;
 
 import org.junit.After;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-import org.junit.rules.TestWatcher;
-import org.junit.runner.Description;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
 
+/**
+ * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports
+ * junit4 format.
+ */
+@ExtendWith(AcceptanceTestBaseTestWatcher.class)
 public class AcceptanceTestBase {
 
   private static final Logger LOG = LoggerFactory.getLogger(AcceptanceTestBase.class);
@@ -131,8 +131,6 @@ protected AcceptanceTestBase() {
     exitedSuccessfully = new ExitedWithCode(0);
   }
 
-  @Rule public final TestName name = new TestName();
-
   @After
   public void tearDownAcceptanceTestBase() {
     reportMemory();
@@ -178,49 +176,6 @@ private void printOutput(final Process process) {
     }
   }
 
-  @Rule
-  public TestWatcher logEraser =
-      new TestWatcher() {
-
-        @Override
-        protected void starting(final Description description) {
-          MDC.put("test", description.getMethodName());
-          MDC.put("class", description.getClassName());
-
-          final String errorMessage = "Uncaught exception in thread \"{}\"";
-          Thread.currentThread()
-              .setUncaughtExceptionHandler(
-                  (thread, error) -> LOG.error(errorMessage, thread.getName(), error));
-          Thread.setDefaultUncaughtExceptionHandler(
-              (thread, error) -> LOG.error(errorMessage, thread.getName(), error));
-        }
-
-        @Override
-        protected void failed(final Throwable e, final Description description) {
-          // add the result at the end of the log so it is self-sufficient
-          LOG.error(
-              "==========================================================================================");
-          LOG.error("Test failed. Reported Throwable at the point of failure:", e);
-          LOG.error(e.getMessage());
-        }
-
-        @Override
-        protected void succeeded(final Description description) {
-          // if so configured, delete logs of successful tests
-          if (!Boolean.getBoolean("acctests.keepLogsOfPassingTests")) {
-            String pathname =
-                "build/acceptanceTestLogs/"
-                    + description.getClassName()
-                    + "."
-                    + description.getMethodName()
-                    + ".log";
-            LOG.info("Test successful, deleting log at {}", pathname);
-            File file = new File(pathname);
-            file.delete();
-          }
-        }
-      };
-
   protected void waitForBlockHeight(final Node node, final long blockchainHeight) {
     WaitUtils.waitFor(
         120,

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/AcceptanceTestBaseJunit5.java

@@ -0,0 +1,200 @@
+/*
+ * Copyright contributors to Hyperledger Besu.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.hyperledger.besu.tests.acceptance.dsl;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.account.Accounts;
+import org.hyperledger.besu.tests.acceptance.dsl.blockchain.Blockchain;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.admin.AdminConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.bft.BftConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.clique.CliqueConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.eth.EthConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.login.LoginConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.net.NetConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.perm.PermissioningConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.priv.PrivConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.process.ExitedWithCode;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.txpool.TxPoolConditions;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.web3.Web3Conditions;
+import org.hyperledger.besu.tests.acceptance.dsl.contract.ContractVerifier;
+import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
+import org.hyperledger.besu.tests.acceptance.dsl.node.cluster.Cluster;
+import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeFactory;
+import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.permissioning.PermissionedNodeBuilder;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.account.AccountTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.admin.AdminTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.bft.BftTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.clique.CliqueTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.contract.ContractTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.miner.MinerTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.net.NetTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.perm.PermissioningTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.privacy.PrivacyTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.txpool.TxPoolTransactions;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.web3.Web3Transactions;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.logging.log4j.ThreadContext;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Superclass for acceptance tests. For now (transition to junit5 is ongoing) this class supports
+ * junit5 format. Once the transition is complete, this class can be removed and recombined with
+ * AcceptanceTestBase (original).
+ */
+@ExtendWith(AcceptanceTestBaseTestWatcher.class)
+public class AcceptanceTestBaseJunit5 {
+
+  private static final Logger LOG = LoggerFactory.getLogger(AcceptanceTestBaseJunit5.class);
+
+  protected final Accounts accounts;
+  protected final AccountTransactions accountTransactions;
+  protected final AdminConditions admin;
+  protected final AdminTransactions adminTransactions;
+  protected final Blockchain blockchain;
+  protected final CliqueConditions clique;
+  protected final CliqueTransactions cliqueTransactions;
+  protected final Cluster cluster;
+  protected final ContractVerifier contractVerifier;
+  protected final ContractTransactions contractTransactions;
+  protected final EthConditions eth;
+  protected final EthTransactions ethTransactions;
+  protected final BftTransactions bftTransactions;
+  protected final BftConditions bft;
+  protected final LoginConditions login;
+  protected final NetConditions net;
+  protected final BesuNodeFactory besu;
+  protected final PermissioningConditions perm;
+  protected final PermissionedNodeBuilder permissionedNodeBuilder;
+  protected final PermissioningTransactions permissioningTransactions;
+  protected final MinerTransactions minerTransactions;
+  protected final Web3Conditions web3;
+  protected final PrivConditions priv;
+  protected final PrivacyTransactions privacyTransactions;
+  protected final TxPoolConditions txPoolConditions;
+  protected final TxPoolTransactions txPoolTransactions;
+  protected final ExitedWithCode exitedSuccessfully;
+
+  private final ExecutorService outputProcessorExecutor = Executors.newCachedThreadPool();
+
+  protected AcceptanceTestBaseJunit5() {
+    ethTransactions = new EthTransactions();
+    accounts = new Accounts(ethTransactions);
+    adminTransactions = new AdminTransactions();
+    cliqueTransactions = new CliqueTransactions();
+    bftTransactions = new BftTransactions();
+    accountTransactions = new AccountTransactions(accounts);
+    permissioningTransactions = new PermissioningTransactions();
+    privacyTransactions = new PrivacyTransactions();
+    contractTransactions = new ContractTransactions();
+    minerTransactions = new MinerTransactions();
+
+    blockchain = new Blockchain(ethTransactions);
+    clique = new CliqueConditions(ethTransactions, cliqueTransactions);
+    eth = new EthConditions(ethTransactions);
+    bft = new BftConditions(bftTransactions);
+    login = new LoginConditions();
+    net = new NetConditions(new NetTransactions());
+    cluster = new Cluster(net);
+    perm = new PermissioningConditions(permissioningTransactions);
+    priv = new PrivConditions(privacyTransactions);
+    admin = new AdminConditions(adminTransactions);
+    web3 = new Web3Conditions(new Web3Transactions());
+    besu = new BesuNodeFactory();
+    txPoolTransactions = new TxPoolTransactions();
+    txPoolConditions = new TxPoolConditions(txPoolTransactions);
+    contractVerifier = new ContractVerifier(accounts.getPrimaryBenefactor());
+    permissionedNodeBuilder = new PermissionedNodeBuilder();
+    exitedSuccessfully = new ExitedWithCode(0);
+  }
+
+  @BeforeEach
+  public void setUp(final TestInfo testInfo) {
+    // log4j is configured to create a file per test
+    // build/acceptanceTestLogs/${ctx:class}.${ctx:test}.log
+    ThreadContext.put("class", this.getClass().getSimpleName());
+    ThreadContext.put("test", testInfo.getTestMethod().get().getName());
+  }
+
+  @AfterEach
+  public void tearDownAcceptanceTestBase() {
+    reportMemory();
+    cluster.close();
+  }
+
+  public void reportMemory() {
+    String os = System.getProperty("os.name");
+    String[] command = null;
+    if (os.contains("Linux")) {
+      command = new String[] {"/usr/bin/top", "-n", "1", "-o", "%MEM", "-b", "-c", "-w", "180"};
+    }
+    if (os.contains("Mac")) {
+      command = new String[] {"/usr/bin/top", "-l", "1", "-o", "mem", "-n", "20"};
+    }
+    if (command != null) {
+      LOG.info("Memory usage at end of test:");
+      final ProcessBuilder processBuilder =
+          new ProcessBuilder(command)
+              .redirectErrorStream(true)
+              .redirectInput(ProcessBuilder.Redirect.INHERIT);
+      try {
+        final Process memInfoProcess = processBuilder.start();
+        outputProcessorExecutor.execute(() -> printOutput(memInfoProcess));
+        memInfoProcess.waitFor();
+        LOG.debug("Memory info process exited with code {}", memInfoProcess.exitValue());
+      } catch (final Exception e) {
+        LOG.warn("Error running memory information process", e);
+      }
+    } else {
+      LOG.info("Don't know how to report memory for OS {}", os);
+    }
+  }
+
+  private void printOutput(final Process process) {
+    try (final BufferedReader in =
+        new BufferedReader(new InputStreamReader(process.getInputStream(), UTF_8))) {
+      String line = in.readLine();
+      while (line != null) {
+        LOG.info(line);
+        line = in.readLine();
+      }
+    } catch (final IOException e) {
+      LOG.warn("Failed to read output from memory information process: ", e);
+    }
+  }
+
+  protected void waitForBlockHeight(final Node node, final long blockchainHeight) {
+    WaitUtils.waitFor(
+        120,
+        () ->
+            assertThat(node.execute(ethTransactions.blockNumber()))
+                .isGreaterThanOrEqualTo(BigInteger.valueOf(blockchainHeight)));
+  }
+}