Burn in candidate for 23.7.3

.github/workflows/checks.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   spotless:
-    runs-on: [besu,Linux,self-hosted,X64,nodocker]
+    runs-on: [besu-research-ubuntu-16]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo
@@ -21,7 +21,7 @@ jobs:
       - name: spotless
         run: ./gradlew --no-daemon --parallel clean spotlessCheck
   javadoc_17:
-    runs-on: [besu,Linux,self-hosted,X64,nodocker]
+    runs-on: [besu-research-ubuntu-8]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: [besu,Linux,self-hosted,X64,nodocker]
+    runs-on: [besu-research-ubuntu-16]
     permissions:
       actions: read
       contents: read

.github/workflows/dco-merge-group.yml

@@ -4,7 +4,7 @@ on:
 
 jobs:
   dco:
-    runs-on: [besu,Linux,self-hosted,nodocker]
+    runs-on: [besu-research-ubuntu-8]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
-      - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"
+      - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"

.github/workflows/dco.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   dco:
-    runs-on: [besu,Linux,self-hosted,nodocker]
+    runs-on: [besu-research-ubuntu-8]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - run: echo "This DCO job runs on pull_request event and workflow_dispatch"
@@ -17,4 +17,4 @@ jobs:
       - name: DCO Check
         uses: tim-actions/dco@v1.1.0
         with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/gradle-wrapper-validation.yml

@@ -5,7 +5,7 @@ on: [push, pull_request]
 jobs:
   validation:
     name: "Gradle Wrapper Validation"
-    runs-on: [besu,Linux,self-hosted,nodocker]
+    runs-on: [besu-research-ubuntu-8]
     steps:
       - uses: actions/checkout@v3
       - uses: gradle/wrapper-validation-action@v1

.github/workflows/pr-checklist-on-open.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   checklist:
     name: "add checklist as a comment on newly opened PRs"
-    runs-on: [besu,Linux,self-hosted,nodocker]
+    runs-on: [besu-research-ubuntu-8]
     steps:
       - uses: actions/github-script@v5
         with:

.github/workflows/release.yml

@@ -4,7 +4,7 @@ on:
     types: released
 jobs:
   dockerPromoteX64:
-    runs-on: [besu,Linux,self-hosted,nodocker]
+    runs-on: [besu-research-ubuntu-16]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3

.github/workflows/repolinter.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   build:
-    runs-on: [besu,Linux,self-hosted,X64,docker]
+    runs-on: [besu-research-ubuntu-16]
     container: ghcr.io/todogroup/repolinter:v0.10.1
     steps:
       - name: Checkout Code

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 23.7.3
+
+### Breaking Changes
+- Removed support for Kotti network (ETC) [#5816](https://github.com/hyperledger/besu/pull/5816)
+- Layered transaction pool implementation is now stable and enabled by default, so the following changes to experimental options have been done [#5772](https://github.com/hyperledger/besu):
+    - `--Xlayered-tx-pool` is gone, to select the implementation use the new `--tx-pool` option with values `layered` (default) or `legacy`
+    - `--Xlayered-tx-pool-layer-max-capacity`, `--Xlayered-tx-pool-max-prioritized` and `--Xlayered-tx-pool-max-future-by-sender` just drop the `X` and keep the same behavior
+
+### Additions and Improvements
+- Add access to an immutable world view to start/end transaction hooks in the tracing API[#5836](https://github.com/hyperledger/besu/pull/5836)
+- Layered transaction pool implementation is now stable and enabled by default. If you want still to use the legacy implementation, use `--tx-pool=legacy` [#5772](https://github.com/hyperledger/besu)
+- Tune G1GC to reduce Besu memory footprint, and new `besu-untuned` start scripts to run without any specific G1GC flags [#5879](https://github.com/hyperledger/besu/pull/5879)
+- Update Holešky config for re-launch [#5890](https://github.com/hyperledger/besu/pull/5890)
+
+### Bug Fixes
+- do not create ignorable storage on revert storage-variables subcommand [#5830](https://github.com/hyperledger/besu/pull/5830) 
+- fix duplicate key errors in EthScheduler-Transactions [#5857](https://github.com/hyperledger/besu/pull/5857)
+
+### Download Links
+
+
 ## 23.7.2
 
 ### Additions and Improvements
@@ -23,8 +44,8 @@
 - Fixing snapsync issue with forest during the heal step [#5776](https://github.com/hyperledger/besu/pull/5776)
 
 ### Download Links
-https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.7.2/besu-23.7.2.tar.gz / sha256: TBA
-https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.7.2/besu-23.7.2.zip / sha256: TBA
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.7.2/besu-23.7.2.tar.gz / sha256: f74b32c1a343cbad90a88aa59276b4c5eefea4643ee542aba2bbf898f85ae242
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.7.2/besu-23.7.2.zip / sha256: a233c83591fc277e3d1530c84bb5ea896abad717d796b5e3b856c79199132b75
 
 
 ## 23.7.1

README.md

@@ -28,7 +28,7 @@ If you have any questions, queries or comments, [Besu channel on Hyperledger Dis
 
 ## Besu Users
 
-To install the Besu binary, follow [these instructions](https://besu.hyperledger.org/HowTo/Get-Started/Install-Binaries/).    
+To install the Besu binary, follow [these instructions](https://besu.hyperledger.org/public-networks/get-started/install/binary-distribution).    
 
 ## Besu Developers
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -46,7 +46,6 @@
 import org.hyperledger.besu.chainimport.RlpBlockImporter;
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.config.NetworkName;
-import org.hyperledger.besu.cli.converter.FractionConverter;
 import org.hyperledger.besu.cli.converter.MetricCategoryConverter;
 import org.hyperledger.besu.cli.converter.PercentageConverter;
 import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty;
@@ -59,6 +58,7 @@
 import org.hyperledger.besu.cli.options.stable.LoggingLevelOption;
 import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption;
 import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions;
+import org.hyperledger.besu.cli.options.stable.TransactionPoolOptions;
 import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions;
 import org.hyperledger.besu.cli.options.unstable.DnsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
@@ -73,7 +73,6 @@
 import org.hyperledger.besu.cli.options.unstable.PrivacyPluginOptions;
 import org.hyperledger.besu.cli.options.unstable.RPCOptions;
 import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions;
-import org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions;
 import org.hyperledger.besu.cli.presynctasks.PreSynchronizationTaskRunner;
 import org.hyperledger.besu.cli.presynctasks.PrivateDatabaseMigrationPreSyncTask;
 import org.hyperledger.besu.cli.subcommands.PasswordSubCommand;
@@ -127,6 +126,7 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
+import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.FrontierTargetingGasLimitCalculator;
 import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration;
@@ -280,7 +280,9 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   final SynchronizerOptions unstableSynchronizerOptions = SynchronizerOptions.create();
   final EthProtocolOptions unstableEthProtocolOptions = EthProtocolOptions.create();
   final MetricsCLIOptions unstableMetricsCLIOptions = MetricsCLIOptions.create();
-  final TransactionPoolOptions unstableTransactionPoolOptions = TransactionPoolOptions.create();
+  final org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions
+      unstableTransactionPoolOptions =
+          org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions.create();
   private final DnsOptions unstableDnsOptions = DnsOptions.create();
   private final MiningOptions unstableMiningOptions = MiningOptions.create();
   private final NatOptions unstableNatOptions = NatOptions.create();
@@ -298,6 +300,10 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
       NodePrivateKeyFileOption.create();
   private final LoggingLevelOption loggingLevelOption = LoggingLevelOption.create();
 
+  @CommandLine.ArgGroup(validate = false, heading = "@|bold Tx Pool Common Options|@%n")
+  final org.hyperledger.besu.cli.options.stable.TransactionPoolOptions
+      stableTransactionPoolOptions = TransactionPoolOptions.create();
+
   private final RunnerBuilder runnerBuilder;
   private final BesuController.Builder controllerBuilderFactory;
   private final BesuPluginContextImpl besuPluginContext;
@@ -454,10 +460,8 @@ static class P2PDiscoveryOptionGroup {
             "The maximum percentage of P2P connections that can be initiated remotely. Must be between 0 and 100 inclusive. (default: ${DEFAULT-VALUE})",
         arity = "1",
         converter = PercentageConverter.class)
-    private final Integer maxRemoteConnectionsPercentage =
-        Fraction.fromFloat(DEFAULT_FRACTION_REMOTE_WIRE_CONNECTIONS_ALLOWED)
-            .toPercentage()
-            .getValue();
+    private final Percentage maxRemoteConnectionsPercentage =
+        Fraction.fromFloat(DEFAULT_FRACTION_REMOTE_WIRE_CONNECTIONS_ALLOWED).toPercentage();
 
     @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
     @CommandLine.Option(
@@ -1111,13 +1115,6 @@ static class MinerOptionGroup {
       arity = "1")
   private final Wei minTransactionGasPrice = DEFAULT_MIN_TRANSACTION_GAS_PRICE;
 
-  @Option(
-      names = {"--rpc-tx-feecap"},
-      description =
-          "Maximum transaction fees (in Wei) accepted for transaction submitted through RPC (default: ${DEFAULT-VALUE})",
-      arity = "1")
-  private final Wei txFeeCap = DEFAULT_RPC_TX_FEE_CAP;
-
   @Option(
       names = {"--min-block-occupancy-ratio"},
       description = "Minimum occupancy ratio for a mined block (default: ${DEFAULT-VALUE})",
@@ -1209,75 +1206,6 @@ static class PermissionsOptionGroup {
           "Sets target gas limit per block. If set, each block's gas limit will approach this setting over time if the current gas limit is different.")
   private final Long targetGasLimit = null;
 
-  // Tx Pool Option Group
-  @CommandLine.ArgGroup(validate = false, heading = "@|bold Tx Pool Options|@%n")
-  TxPoolOptionGroup txPoolOptionGroup = new TxPoolOptionGroup();
-
-  static class TxPoolOptionGroup {
-    @CommandLine.Option(
-        names = {"--tx-pool-disable-locals"},
-        paramLabel = "<Boolean>",
-        description =
-            "Set to true if transactions sent via RPC should have the same checks and not be prioritized over remote ones (default: ${DEFAULT-VALUE})",
-        fallbackValue = "true",
-        arity = "0..1")
-    private Boolean disableLocalTxs = TransactionPoolConfiguration.DEFAULT_DISABLE_LOCAL_TXS;
-
-    @CommandLine.Option(
-        names = {"--tx-pool-enable-save-restore"},
-        paramLabel = "<Boolean>",
-        description =
-            "Set to true to enable saving the txpool content to file on shutdown and reloading it on startup (default: ${DEFAULT-VALUE})",
-        fallbackValue = "true",
-        arity = "0..1")
-    private Boolean saveRestoreEnabled = TransactionPoolConfiguration.DEFAULT_ENABLE_SAVE_RESTORE;
-
-    @CommandLine.Option(
-        names = {"--tx-pool-limit-by-account-percentage"},
-        paramLabel = "<DOUBLE>",
-        converter = FractionConverter.class,
-        description =
-            "Maximum portion of the transaction pool which a single account may occupy with future transactions (default: ${DEFAULT-VALUE})",
-        arity = "1")
-    private Float txPoolLimitByAccountPercentage =
-        TransactionPoolConfiguration.DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE;
-
-    @CommandLine.Option(
-        names = {"--tx-pool-save-file"},
-        paramLabel = "<STRING>",
-        description =
-            "If saving the txpool content is enabled, define a custom path for the save file (default: ${DEFAULT-VALUE} in the data-dir)",
-        arity = "1")
-    private File saveFile = TransactionPoolConfiguration.DEFAULT_SAVE_FILE;
-
-    @Option(
-        names = {"--tx-pool-max-size"},
-        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
-        description =
-            "Maximum number of pending transactions that will be kept in the transaction pool (default: ${DEFAULT-VALUE})",
-        arity = "1")
-    private final Integer txPoolMaxSize =
-        TransactionPoolConfiguration.DEFAULT_MAX_PENDING_TRANSACTIONS;
-
-    @Option(
-        names = {"--tx-pool-retention-hours"},
-        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
-        description =
-            "Maximum retention period of pending transactions in hours (default: ${DEFAULT-VALUE})",
-        arity = "1")
-    private final Integer pendingTxRetentionPeriod =
-        TransactionPoolConfiguration.DEFAULT_TX_RETENTION_HOURS;
-
-    @Option(
-        names = {"--tx-pool-price-bump"},
-        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
-        converter = PercentageConverter.class,
-        description =
-            "Price bump percentage to replace an already existing transaction  (default: ${DEFAULT-VALUE})",
-        arity = "1")
-    private final Integer priceBump = TransactionPoolConfiguration.DEFAULT_PRICE_BUMP.getValue();
-  }
-
   @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
   @Option(
       names = {"--key-value-storage"},
@@ -1906,10 +1834,15 @@ private void validateOptions() {
     validateRpcOptionsParams();
     validateChainDataPruningParams();
     validatePostMergeCheckpointBlockRequirements();
+    validateTransactionPoolOptions();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
 
+  private void validateTransactionPoolOptions() {
+    stableTransactionPoolOptions.validate(commandLine);
+  }
+
   private void validateRequiredOptions() {
     commandLine
         .getCommandSpec()
@@ -2987,28 +2920,14 @@ private SynchronizerConfiguration buildSyncConfig() {
   }
 
   private TransactionPoolConfiguration buildTransactionPoolConfiguration() {
-    return unstableTransactionPoolOptions
-        .toDomainObject()
-        .enableSaveRestore(txPoolOptionGroup.saveRestoreEnabled)
-        .disableLocalTransactions(txPoolOptionGroup.disableLocalTxs)
-        .txPoolLimitByAccountPercentage(txPoolOptionGroup.txPoolLimitByAccountPercentage)
-        .txPoolMaxSize(txPoolOptionGroup.txPoolMaxSize)
-        .pendingTxRetentionPeriod(txPoolOptionGroup.pendingTxRetentionPeriod)
-        .priceBump(Percentage.fromInt(txPoolOptionGroup.priceBump))
-        .txFeeCap(txFeeCap)
-        .saveFile(dataPath.resolve(txPoolOptionGroup.saveFile.getPath()).toFile())
+    final var stableTxPoolOption = stableTransactionPoolOptions.toDomainObject();
+    return ImmutableTransactionPoolConfiguration.builder()
+        .from(stableTxPoolOption)
+        .unstable(unstableTransactionPoolOptions.toDomainObject())
+        .saveFile((dataPath.resolve(stableTxPoolOption.getSaveFile().getPath()).toFile()))
         .build();
   }
 
-  /**
-   * Return the file where to save txpool content if the relative option is enabled.
-   *
-   * @return the save file
-   */
-  public File getSaveFile() {
-    return txPoolOptionGroup.saveFile;
-  }
-
   private boolean isPruningEnabled() {
     return pruningEnabled;
   }
@@ -3489,7 +3408,8 @@ private void setMergeConfigOptions() {
         getActualGenesisConfigOptions().getTerminalTotalDifficulty().isPresent());
   }
 
-  private void setIgnorableStorageSegments() {
+  /** Set ignorable segments in RocksDB Storage Provider plugin. */
+  public void setIgnorableStorageSegments() {
     if (!unstableChainPruningOptions.getChainDataPruningEnabled()) {
       rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE);
     }
@@ -3612,9 +3532,7 @@ private String generateConfigurationOverview() {
       builder.setHighSpecEnabled();
     }
 
-    if (buildTransactionPoolConfiguration().getLayeredTxPoolEnabled()) {
-      builder.setLayeredTxPoolEnabled();
-    }
+    builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
 
     return builder.build();
   }