Merge remote-tracking branch 'upstream/main' into feature/reduce-rece…

…ipt-size

.circleci/config.yml

@@ -233,16 +233,16 @@ jobs:
       - capture_test_logs
 
   acceptanceTestsCliqueBft:
-    executor: besu_executor_xl
+    executor: besu_executor_med
     steps:
       - prepare
       - attach_workspace:
           at: ~/project
       - run:
           name: AcceptanceTests (Non-Mainnet)
-          no_output_timeout: 20m
+          no_output_timeout: 30m
           command: |
-            ./gradlew --no-daemon acceptanceTestCliqueBft
+            ./gradlew --no-daemon --max-workers=1 acceptanceTestCliqueBft
       - capture_test_results
       - capture_test_logs
 
@@ -269,7 +269,7 @@ jobs:
       - capture_test_logs
 
   acceptanceTestsPermissioning:
-    executor: besu_executor_xl
+    executor: besu_executor_med
     steps:
       - prepare
       - attach_workspace:
@@ -278,7 +278,7 @@ jobs:
           name: AcceptanceTests (Non-Mainnet)
           no_output_timeout: 20m
           command: |
-            ./gradlew --no-daemon acceptanceTestPermissioning
+            ./gradlew --no-daemon --max-workers=1 acceptanceTestPermissioning
       - capture_test_results
       - capture_test_logs
 
@@ -428,12 +428,12 @@ workflows:
       - acceptanceTestsPermissioning:
           requires:
             - assemble
-            - acceptanceTestsCliqueBft
       - buildDocker:
           requires:
             - assemble
       - buildArm64Docker:
           requires:
+            - buildDocker
             - assemble
       - publish:
           filters:

CHANGELOG.md

@@ -5,17 +5,24 @@
 ### Breaking Changes
 - New `EXECUTION_HALTED` error returned if there is an error executing or simulating a transaction, with the reason for execution being halted. Replaces the generic `INTERNAL_ERROR` return code in certain cases which some applications may be checking for [#6343](https://github.com/hyperledger/besu/pull/6343)
 - The Besu Docker images with `openjdk-latest` tags since 23.10.3 were incorrectly using UID 1001 instead of 1000 for the container's `besu` user. The user now uses 1000 again. Containers created from or migrated to images using UID 1001 will need to chown their persistent database files to UID 1000 [#6360](https://github.com/hyperledger/besu/pull/6360)
+- The deprecated `--privacy-onchain-groups-enabled` option has now been removed. Use the `--privacy-flexible-groups-enabled` option instead. [#6411](https://github.com/hyperledger/besu/pull/6411)
 
 ### Deprecations
 
 ### Additions and Improvements
 - Optimize RocksDB WAL files, allows for faster restart and a more linear disk space utilization [#6328](https://github.com/hyperledger/besu/pull/6328)
 - Disable transaction handling when the node is not in sync, to avoid unnecessary transaction validation work [#6302](https://github.com/hyperledger/besu/pull/6302)
+- Introduce TransactionEvaluationContext to pass data between transaction selectors and plugin, during block creation [#6381](https://github.com/hyperledger/besu/pull/6381) 
+- Upgrade dependencies [#6377](https://github.com/hyperledger/besu/pull/6377)
+- Upgrade `com.fasterxml.jackson` dependencies [#6378](https://github.com/hyperledger/besu/pull/6378) 
+- Upgrade Guava dependency [#6396](https://github.com/hyperledger/besu/pull/6396)
+- Upgrade Mockito [#6397](https://github.com/hyperledger/besu/pull/6397)
 
 ### Bug fixes
 - INTERNAL_ERROR from `eth_estimateGas` JSON/RPC calls [#6344](https://github.com/hyperledger/besu/issues/6344)
 - Fix Besu Docker images with `openjdk-latest` tags since 23.10.3 using UID 1001 instead of 1000 for the `besu` user [#6360](https://github.com/hyperledger/besu/pull/6360)
 - Fluent EVM API definition for Tangerine Whistle had incorrect code size validation configured [#6382](https://github.com/hyperledger/besu/pull/6382)
+- Correct mining beneficiary for Clique networks in TraceServiceImpl [#6390](https://github.com/hyperledger/besu/pull/6390)
 
 ### Download Links
 
@@ -25,7 +32,7 @@
 ### Breaking Changes
 
 ### Deprecations
-- Forest pruning (`pruning-enabled` options) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230)
+- Forest pruning (`pruning-enabled` option) is deprecated and will be removed soon. To save disk space consider switching to Bonsai data storage format [#6230](https://github.com/hyperledger/besu/pull/6230)
 
 ### Additions and Improvements
 - Add error messages on authentication failures with username and password [#6212](https://github.com/hyperledger/besu/pull/6212)

SUPPORT.md

@@ -7,7 +7,7 @@ Welcome to the Besu repository! The following links are a set of guidelines for
 Having Github, Discord, and Linux Foundation accounts is necessary for obtaining support for Besu through the community channels, wiki and issue management.  
 * If you want to raise an issue, you can do so [on the github issue tab](https://github.com/hyperledger/besu/issues).   
 * Hyperledger Discord requires a [Discord account].
-* The Hyperlegder wiki also requires a [Linux Foundation (LF) account] in order to edit pages.
+* The Hyperledger wiki also requires a [Linux Foundation (LF) account] in order to edit pages.
 
 ### Useful support links
 

acceptance-tests/dsl/build.gradle

@@ -26,7 +26,6 @@ dependencies {
   implementation project(':testutil')
   implementation project(':util')
 
-  implementation 'com.github.tomakehurst:wiremock-jre8'
   implementation 'com.google.guava:guava'
   implementation 'com.google.dagger:dagger'
   annotationProcessor 'com.google.dagger:dagger-compiler'
@@ -45,6 +44,7 @@ dependencies {
   implementation 'org.web3j:abi'
   implementation 'org.web3j:besu'
   implementation 'org.web3j:crypto'
+  implementation 'org.wiremock:wiremock'
 
   implementation 'org.testcontainers:testcontainers'
   implementation 'org.junit.jupiter:junit-jupiter'

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/EthConditions.java

@@ -77,6 +77,10 @@ public Condition miningStatus(final boolean isMining) {
     return new MiningStatusCondition(transactions.mining(), isMining);
   }
 
+  public Condition syncingStatus(final boolean isSyncing) {
+    return new SyncingStatusCondition(transactions.syncing(), isSyncing);
+  }
+
   public Condition expectNewPendingTransactions(
       final BigInteger filterId, final List<String> transactionHashes) {
     return new NewPendingTransactionFilterChangesCondition(

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/condition/eth/SyncingStatusCondition.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.tests.acceptance.dsl.condition.eth;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.hyperledger.besu.tests.acceptance.dsl.WaitUtils;
+import org.hyperledger.besu.tests.acceptance.dsl.condition.Condition;
+import org.hyperledger.besu.tests.acceptance.dsl.node.Node;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.eth.EthSyncingTransaction;
+
+public class SyncingStatusCondition implements Condition {
+
+  private final EthSyncingTransaction transaction;
+  private final boolean syncingMiningStatus;
+
+  public SyncingStatusCondition(
+      final EthSyncingTransaction transaction, final boolean syncingStatus) {
+    this.transaction = transaction;
+    this.syncingMiningStatus = syncingStatus;
+  }
+
+  @Override
+  public void verify(final Node node) {
+    WaitUtils.waitFor(
+        10, () -> assertThat(node.execute(transaction)).isEqualTo(syncingMiningStatus));
+  }
+}

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -29,6 +29,7 @@
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Util;
+import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration;
 import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
@@ -100,6 +101,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
 
   private final String name;
   private MiningParameters miningParameters;
+  private TransactionPoolConfiguration txPoolConfiguration;
   private final List<String> runCommand;
   private PrivacyParameters privacyParameters = PrivacyParameters.DEFAULT;
   private final JsonRpcConfiguration jsonRpcConfiguration;
@@ -135,6 +137,7 @@ public BesuNode(
       final String name,
       final Optional<Path> dataPath,
       final MiningParameters miningParameters,
+      final TransactionPoolConfiguration txPoolConfiguration,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final Optional<JsonRpcConfiguration> engineRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
@@ -184,6 +187,7 @@ public BesuNode(
         () -> this.keyPair = KeyPairUtil.loadKeyPair(homeDirectory));
     this.name = name;
     this.miningParameters = miningParameters;
+    this.txPoolConfiguration = txPoolConfiguration;
     this.jsonRpcConfiguration = jsonRpcConfiguration;
     this.engineRpcConfiguration = engineRpcConfiguration;
     this.webSocketConfiguration = webSocketConfiguration;
@@ -661,14 +665,23 @@ public void setBootnodes(final List<URI> bootnodes) {
     this.bootnodes.addAll(bootnodes);
   }
 
-  MiningParameters getMiningParameters() {
+  public MiningParameters getMiningParameters() {
     return miningParameters;
   }
 
   public void setMiningParameters(final MiningParameters miningParameters) {
     this.miningParameters = miningParameters;
   }
 
+  public TransactionPoolConfiguration getTransactionPoolConfiguration() {
+    return txPoolConfiguration;
+  }
+
+  public void setTransactionPoolConfiguration(
+      final TransactionPoolConfiguration txPoolConfiguration) {
+    this.txPoolConfiguration = txPoolConfiguration;
+  }
+
   public PrivacyParameters getPrivacyParameters() {
     return privacyParameters;
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -17,8 +17,10 @@
 import static com.google.common.base.Preconditions.checkState;
 import static java.nio.charset.StandardCharsets.UTF_8;
 
+import org.hyperledger.besu.cli.options.TransactionPoolOptions;
 import org.hyperledger.besu.cli.options.unstable.NetworkingOptions;
 import org.hyperledger.besu.ethereum.api.jsonrpc.ipc.JsonRpcIpcConfiguration;
+import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
@@ -98,6 +100,15 @@ public void startNode(final BesuNode node) {
     params.add("--p2p-port");
     params.add(node.getP2pPort());
 
+    params.addAll(
+        TransactionPoolOptions.fromConfig(
+                ImmutableTransactionPoolConfiguration.builder()
+                    .from(node.getTransactionPoolConfiguration())
+                    .strictTransactionReplayProtectionEnabled(
+                        node.isStrictTxReplayProtectionEnabled())
+                    .build())
+            .getCLIOptions());
+
     if (node.getMiningParameters().isMiningEnabled()) {
       params.add("--miner-enabled");
       params.add("--miner-coinbase");
@@ -391,9 +402,6 @@ public void startNode(final BesuNode node) {
     params.add("--auto-log-bloom-caching-enabled");
     params.add("false");
 
-    params.add("--strict-tx-replay-protection-enabled");
-    params.add(Boolean.toString(node.isStrictTxReplayProtectionEnabled()));
-
     final String level = System.getProperty("root.log.level");
     if (level != null) {
       params.add("--logging=" + level);

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -182,6 +182,7 @@ public void startNode(final BesuNode node) {
 
     final TransactionPoolConfiguration txPoolConfig =
         ImmutableTransactionPoolConfiguration.builder()
+            .from(node.getTransactionPoolConfiguration())
             .strictTransactionReplayProtectionEnabled(node.isStrictTxReplayProtectionEnabled())
             .build();
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/cluster/Cluster.java

@@ -86,11 +86,8 @@ public void start(final List<? extends RunnableNode> nodes) {
     final Optional<? extends RunnableNode> bootnode = selectAndStartBootnode(nodes);
 
     nodes.parallelStream()
-        .filter(
-            node -> {
-              LOG.info("starting non-bootnode {}", node.getName());
-              return bootnode.map(boot -> boot != node).orElse(true);
-            })
+        .filter(node -> bootnode.map(boot -> boot != node).orElse(true))
+        .peek(node -> LOG.info("starting non-bootnode {}", node.getName()))
         .forEach(this::startNode);
 
     if (clusterConfiguration.isAwaitPeerDiscovery()) {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java

@@ -22,6 +22,7 @@
 import org.hyperledger.besu.ethereum.api.jsonrpc.websocket.WebSocketConfiguration;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
+import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration;
 import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
@@ -39,6 +40,7 @@ public class BesuNodeConfiguration {
   private final String name;
   private final Optional<Path> dataPath;
   private final MiningParameters miningParameters;
+  private final TransactionPoolConfiguration transactionPoolConfiguration;
   private final JsonRpcConfiguration jsonRpcConfiguration;
   private final Optional<JsonRpcConfiguration> engineRpcConfiguration;
   private final WebSocketConfiguration webSocketConfiguration;
@@ -74,6 +76,7 @@ public class BesuNodeConfiguration {
       final String name,
       final Optional<Path> dataPath,
       final MiningParameters miningParameters,
+      final TransactionPoolConfiguration transactionPoolConfiguration,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final Optional<JsonRpcConfiguration> engineRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
@@ -106,6 +109,7 @@ public class BesuNodeConfiguration {
       final Map<String, String> environment) {
     this.name = name;
     this.miningParameters = miningParameters;
+    this.transactionPoolConfiguration = transactionPoolConfiguration;
     this.jsonRpcConfiguration = jsonRpcConfiguration;
     this.engineRpcConfiguration = engineRpcConfiguration;
     this.webSocketConfiguration = webSocketConfiguration;
@@ -147,6 +151,10 @@ public MiningParameters getMiningParameters() {
     return miningParameters;
   }
 
+  public TransactionPoolConfiguration getTransactionPoolConfiguration() {
+    return transactionPoolConfiguration;
+  }
+
   public JsonRpcConfiguration getJsonRpcConfiguration() {
     return jsonRpcConfiguration;
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java

@@ -35,6 +35,7 @@
 import org.hyperledger.besu.ethereum.core.ImmutableMiningParameters.MutableInitValues;
 import org.hyperledger.besu.ethereum.core.MiningParameters;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
+import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.p2p.config.NetworkingConfiguration;
 import org.hyperledger.besu.ethereum.p2p.rlpx.connections.netty.TLSConfiguration;
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
@@ -63,7 +64,8 @@ public class BesuNodeConfigurationBuilder {
           .mutableInitValues(
               MutableInitValues.builder().coinbase(AddressHelpers.ofValue(1)).build())
           .build();
-
+  private TransactionPoolConfiguration transactionPoolConfiguration =
+      TransactionPoolConfiguration.DEFAULT;
   private JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault();
   private JsonRpcConfiguration engineRpcConfiguration = JsonRpcConfiguration.createEngineDefault();
   private WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault();
@@ -128,6 +130,12 @@ public BesuNodeConfigurationBuilder miningConfiguration(final MiningParameters m
     return this;
   }
 
+  public BesuNodeConfigurationBuilder transactionPoolConfiguration(
+      final TransactionPoolConfiguration transactionPoolConfiguration) {
+    this.transactionPoolConfiguration = transactionPoolConfiguration;
+    return this;
+  }
+
   public BesuNodeConfigurationBuilder jsonRpcConfiguration(
       final JsonRpcConfiguration jsonRpcConfiguration) {
     this.jsonRpcConfiguration = jsonRpcConfiguration;
@@ -503,6 +511,7 @@ public BesuNodeConfiguration build() {
         name,
         dataPath,
         miningParameters,
+        transactionPoolConfiguration,
         jsonRpcConfiguration,
         Optional.of(engineRpcConfiguration),
         webSocketConfiguration,

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -64,6 +64,7 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException {
         config.getName(),
         config.getDataPath(),
         config.getMiningParameters(),
+        config.getTransactionPoolConfiguration(),
         config.getJsonRpcConfiguration(),
         config.getEngineRpcConfiguration(),
         config.getWebSocketConfiguration(),

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java

@@ -100,6 +100,7 @@ public PrivacyNode(
             besuConfig.getName(),
             besuConfig.getDataPath(),
             besuConfig.getMiningParameters(),
+            besuConfig.getTransactionPoolConfiguration(),
             besuConfig.getJsonRpcConfiguration(),
             besuConfig.getEngineRpcConfiguration(),
             besuConfig.getWebSocketConfiguration(),