Release 23.7.3-RC2 - Holesky Hotfix (#5938)

* Revert "Burn in candidate for 23.7.3 (#5906)"

This reverts commit ed9d80e.

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

* Update holesky with fixed extraData, genesis time, shanghaiTime (#5890)

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

* 23.7.3-RC2 version bump and 23.7.2 release SHAs
Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

* updated gradle verification metadata (#5870)

* removed old artefacts

* works with compileTestJava

* restored metadata needed for codeQL and trusted-artifacts block for javadoc/sources

Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>

---------

Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>

* Add rocksdbjni-8.0.0 gradle verification back in

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>

---------

Signed-off-by: Simon Dudley <simon.dudley@consensys.net>
Signed-off-by: Sally MacFarlane <macfarla.github@gmail.com>
Co-authored-by: Sally MacFarlane <macfarla.github@gmail.com>

.github/workflows/checks.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   spotless:
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: [besu,Linux,self-hosted,X64,nodocker]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo
@@ -21,7 +21,7 @@ jobs:
       - name: spotless
         run: ./gradlew --no-daemon --parallel clean spotlessCheck
   javadoc_17:
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: [besu,Linux,self-hosted,X64,nodocker]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - name: Checkout Repo

.github/workflows/codeql.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   analyze:
     name: Analyze
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: [besu,Linux,self-hosted,X64,nodocker]
     permissions:
       actions: read
       contents: read

.github/workflows/dco-merge-group.yml

@@ -4,7 +4,7 @@ on:
 
 jobs:
   dco:
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: [besu,Linux,self-hosted,nodocker]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
-      - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"
+      - run: echo "This DCO job runs on merge_queue event and doesn't check PR contents"

.github/workflows/dco.yml

@@ -5,7 +5,7 @@ on:
 
 jobs:
   dco:
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: [besu,Linux,self-hosted,nodocker]
     if: ${{ github.actor != 'dependabot[bot]' }}
     steps:
       - run: echo "This DCO job runs on pull_request event and workflow_dispatch"
@@ -17,4 +17,4 @@ jobs:
       - name: DCO Check
         uses: tim-actions/dco@v1.1.0
         with:
-          commits: ${{ steps.get-pr-commits.outputs.commits }}
+          commits: ${{ steps.get-pr-commits.outputs.commits }}

.github/workflows/gradle-wrapper-validation.yml

@@ -5,7 +5,7 @@ on: [push, pull_request]
 jobs:
   validation:
     name: "Gradle Wrapper Validation"
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: [besu,Linux,self-hosted,nodocker]
     steps:
       - uses: actions/checkout@v3
       - uses: gradle/wrapper-validation-action@v1

.github/workflows/pr-checklist-on-open.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   checklist:
     name: "add checklist as a comment on newly opened PRs"
-    runs-on: [besu-research-ubuntu-8]
+    runs-on: [besu,Linux,self-hosted,nodocker]
     steps:
       - uses: actions/github-script@v5
         with:

.github/workflows/release.yml

@@ -4,7 +4,7 @@ on:
     types: released
 jobs:
   dockerPromoteX64:
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: [besu,Linux,self-hosted,nodocker]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3

.github/workflows/repolinter.yml

@@ -15,7 +15,7 @@ on:
 
 jobs:
   build:
-    runs-on: [besu-research-ubuntu-16]
+    runs-on: [besu,Linux,self-hosted,X64,docker]
     container: ghcr.io/todogroup/repolinter:v0.10.1
     steps:
       - name: Checkout Code

CHANGELOG.md

@@ -2,21 +2,8 @@
 
 ## 23.7.3
 
-### Breaking Changes
-- Removed support for Kotti network (ETC) [#5816](https://github.com/hyperledger/besu/pull/5816)
-- Layered transaction pool implementation is now stable and enabled by default, so the following changes to experimental options have been done [#5772](https://github.com/hyperledger/besu):
-    - `--Xlayered-tx-pool` is gone, to select the implementation use the new `--tx-pool` option with values `layered` (default) or `legacy`
-    - `--Xlayered-tx-pool-layer-max-capacity`, `--Xlayered-tx-pool-max-prioritized` and `--Xlayered-tx-pool-max-future-by-sender` just drop the `X` and keep the same behavior
-
 ### Additions and Improvements
-- Add access to an immutable world view to start/end transaction hooks in the tracing API[#5836](https://github.com/hyperledger/besu/pull/5836)
-- Layered transaction pool implementation is now stable and enabled by default. If you want still to use the legacy implementation, use `--tx-pool=legacy` [#5772](https://github.com/hyperledger/besu)
-- Tune G1GC to reduce Besu memory footprint, and new `besu-untuned` start scripts to run without any specific G1GC flags [#5879](https://github.com/hyperledger/besu/pull/5879)
-- Update Holešky config for re-launch [#5890](https://github.com/hyperledger/besu/pull/5890)
-
-### Bug Fixes
-- do not create ignorable storage on revert storage-variables subcommand [#5830](https://github.com/hyperledger/besu/pull/5830) 
-- fix duplicate key errors in EthScheduler-Transactions [#5857](https://github.com/hyperledger/besu/pull/5857)
+- Update Holesky config for re-launch [#5890](https://github.com/hyperledger/besu/pull/5890)
 
 ### Download Links
 

README.md

@@ -28,7 +28,7 @@ If you have any questions, queries or comments, [Besu channel on Hyperledger Dis
 
 ## Besu Users
 
-To install the Besu binary, follow [these instructions](https://besu.hyperledger.org/public-networks/get-started/install/binary-distribution).    
+To install the Besu binary, follow [these instructions](https://besu.hyperledger.org/HowTo/Get-Started/Install-Binaries/).    
 
 ## Besu Developers
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -46,6 +46,7 @@
 import org.hyperledger.besu.chainimport.RlpBlockImporter;
 import org.hyperledger.besu.cli.config.EthNetworkConfig;
 import org.hyperledger.besu.cli.config.NetworkName;
+import org.hyperledger.besu.cli.converter.FractionConverter;
 import org.hyperledger.besu.cli.converter.MetricCategoryConverter;
 import org.hyperledger.besu.cli.converter.PercentageConverter;
 import org.hyperledger.besu.cli.custom.CorsAllowedOriginsProperty;
@@ -58,7 +59,6 @@
 import org.hyperledger.besu.cli.options.stable.LoggingLevelOption;
 import org.hyperledger.besu.cli.options.stable.NodePrivateKeyFileOption;
 import org.hyperledger.besu.cli.options.stable.P2PTLSConfigOptions;
-import org.hyperledger.besu.cli.options.stable.TransactionPoolOptions;
 import org.hyperledger.besu.cli.options.unstable.ChainPruningOptions;
 import org.hyperledger.besu.cli.options.unstable.DnsOptions;
 import org.hyperledger.besu.cli.options.unstable.EthProtocolOptions;
@@ -73,6 +73,7 @@
 import org.hyperledger.besu.cli.options.unstable.PrivacyPluginOptions;
 import org.hyperledger.besu.cli.options.unstable.RPCOptions;
 import org.hyperledger.besu.cli.options.unstable.SynchronizerOptions;
+import org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions;
 import org.hyperledger.besu.cli.presynctasks.PreSynchronizationTaskRunner;
 import org.hyperledger.besu.cli.presynctasks.PrivateDatabaseMigrationPreSyncTask;
 import org.hyperledger.besu.cli.subcommands.PasswordSubCommand;
@@ -126,7 +127,6 @@
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
-import org.hyperledger.besu.ethereum.eth.transactions.ImmutableTransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.ethereum.mainnet.FrontierTargetingGasLimitCalculator;
 import org.hyperledger.besu.ethereum.p2p.config.DiscoveryConfiguration;
@@ -280,9 +280,7 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
   final SynchronizerOptions unstableSynchronizerOptions = SynchronizerOptions.create();
   final EthProtocolOptions unstableEthProtocolOptions = EthProtocolOptions.create();
   final MetricsCLIOptions unstableMetricsCLIOptions = MetricsCLIOptions.create();
-  final org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions
-      unstableTransactionPoolOptions =
-          org.hyperledger.besu.cli.options.unstable.TransactionPoolOptions.create();
+  final TransactionPoolOptions unstableTransactionPoolOptions = TransactionPoolOptions.create();
   private final DnsOptions unstableDnsOptions = DnsOptions.create();
   private final MiningOptions unstableMiningOptions = MiningOptions.create();
   private final NatOptions unstableNatOptions = NatOptions.create();
@@ -300,10 +298,6 @@ public class BesuCommand implements DefaultCommandValues, Runnable {
       NodePrivateKeyFileOption.create();
   private final LoggingLevelOption loggingLevelOption = LoggingLevelOption.create();
 
-  @CommandLine.ArgGroup(validate = false, heading = "@|bold Tx Pool Common Options|@%n")
-  final org.hyperledger.besu.cli.options.stable.TransactionPoolOptions
-      stableTransactionPoolOptions = TransactionPoolOptions.create();
-
   private final RunnerBuilder runnerBuilder;
   private final BesuController.Builder controllerBuilderFactory;
   private final BesuPluginContextImpl besuPluginContext;
@@ -460,8 +454,10 @@ static class P2PDiscoveryOptionGroup {
             "The maximum percentage of P2P connections that can be initiated remotely. Must be between 0 and 100 inclusive. (default: ${DEFAULT-VALUE})",
         arity = "1",
         converter = PercentageConverter.class)
-    private final Percentage maxRemoteConnectionsPercentage =
-        Fraction.fromFloat(DEFAULT_FRACTION_REMOTE_WIRE_CONNECTIONS_ALLOWED).toPercentage();
+    private final Integer maxRemoteConnectionsPercentage =
+        Fraction.fromFloat(DEFAULT_FRACTION_REMOTE_WIRE_CONNECTIONS_ALLOWED)
+            .toPercentage()
+            .getValue();
 
     @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
     @CommandLine.Option(
@@ -1115,6 +1111,13 @@ static class MinerOptionGroup {
       arity = "1")
   private final Wei minTransactionGasPrice = DEFAULT_MIN_TRANSACTION_GAS_PRICE;
 
+  @Option(
+      names = {"--rpc-tx-feecap"},
+      description =
+          "Maximum transaction fees (in Wei) accepted for transaction submitted through RPC (default: ${DEFAULT-VALUE})",
+      arity = "1")
+  private final Wei txFeeCap = DEFAULT_RPC_TX_FEE_CAP;
+
   @Option(
       names = {"--min-block-occupancy-ratio"},
       description = "Minimum occupancy ratio for a mined block (default: ${DEFAULT-VALUE})",
@@ -1206,6 +1209,75 @@ static class PermissionsOptionGroup {
           "Sets target gas limit per block. If set, each block's gas limit will approach this setting over time if the current gas limit is different.")
   private final Long targetGasLimit = null;
 
+  // Tx Pool Option Group
+  @CommandLine.ArgGroup(validate = false, heading = "@|bold Tx Pool Options|@%n")
+  TxPoolOptionGroup txPoolOptionGroup = new TxPoolOptionGroup();
+
+  static class TxPoolOptionGroup {
+    @CommandLine.Option(
+        names = {"--tx-pool-disable-locals"},
+        paramLabel = "<Boolean>",
+        description =
+            "Set to true if transactions sent via RPC should have the same checks and not be prioritized over remote ones (default: ${DEFAULT-VALUE})",
+        fallbackValue = "true",
+        arity = "0..1")
+    private Boolean disableLocalTxs = TransactionPoolConfiguration.DEFAULT_DISABLE_LOCAL_TXS;
+
+    @CommandLine.Option(
+        names = {"--tx-pool-enable-save-restore"},
+        paramLabel = "<Boolean>",
+        description =
+            "Set to true to enable saving the txpool content to file on shutdown and reloading it on startup (default: ${DEFAULT-VALUE})",
+        fallbackValue = "true",
+        arity = "0..1")
+    private Boolean saveRestoreEnabled = TransactionPoolConfiguration.DEFAULT_ENABLE_SAVE_RESTORE;
+
+    @CommandLine.Option(
+        names = {"--tx-pool-limit-by-account-percentage"},
+        paramLabel = "<DOUBLE>",
+        converter = FractionConverter.class,
+        description =
+            "Maximum portion of the transaction pool which a single account may occupy with future transactions (default: ${DEFAULT-VALUE})",
+        arity = "1")
+    private Float txPoolLimitByAccountPercentage =
+        TransactionPoolConfiguration.DEFAULT_LIMIT_TX_POOL_BY_ACCOUNT_PERCENTAGE;
+
+    @CommandLine.Option(
+        names = {"--tx-pool-save-file"},
+        paramLabel = "<STRING>",
+        description =
+            "If saving the txpool content is enabled, define a custom path for the save file (default: ${DEFAULT-VALUE} in the data-dir)",
+        arity = "1")
+    private File saveFile = TransactionPoolConfiguration.DEFAULT_SAVE_FILE;
+
+    @Option(
+        names = {"--tx-pool-max-size"},
+        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
+        description =
+            "Maximum number of pending transactions that will be kept in the transaction pool (default: ${DEFAULT-VALUE})",
+        arity = "1")
+    private final Integer txPoolMaxSize =
+        TransactionPoolConfiguration.DEFAULT_MAX_PENDING_TRANSACTIONS;
+
+    @Option(
+        names = {"--tx-pool-retention-hours"},
+        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
+        description =
+            "Maximum retention period of pending transactions in hours (default: ${DEFAULT-VALUE})",
+        arity = "1")
+    private final Integer pendingTxRetentionPeriod =
+        TransactionPoolConfiguration.DEFAULT_TX_RETENTION_HOURS;
+
+    @Option(
+        names = {"--tx-pool-price-bump"},
+        paramLabel = MANDATORY_INTEGER_FORMAT_HELP,
+        converter = PercentageConverter.class,
+        description =
+            "Price bump percentage to replace an already existing transaction  (default: ${DEFAULT-VALUE})",
+        arity = "1")
+    private final Integer priceBump = TransactionPoolConfiguration.DEFAULT_PRICE_BUMP.getValue();
+  }
+
   @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
   @Option(
       names = {"--key-value-storage"},
@@ -1834,15 +1906,10 @@ private void validateOptions() {
     validateRpcOptionsParams();
     validateChainDataPruningParams();
     validatePostMergeCheckpointBlockRequirements();
-    validateTransactionPoolOptions();
     p2pTLSConfigOptions.checkP2PTLSOptionsDependencies(logger, commandLine);
     pkiBlockCreationOptions.checkPkiBlockCreationOptionsDependencies(logger, commandLine);
   }
 
-  private void validateTransactionPoolOptions() {
-    stableTransactionPoolOptions.validate(commandLine);
-  }
-
   private void validateRequiredOptions() {
     commandLine
         .getCommandSpec()
@@ -2920,14 +2987,28 @@ private SynchronizerConfiguration buildSyncConfig() {
   }
 
   private TransactionPoolConfiguration buildTransactionPoolConfiguration() {
-    final var stableTxPoolOption = stableTransactionPoolOptions.toDomainObject();
-    return ImmutableTransactionPoolConfiguration.builder()
-        .from(stableTxPoolOption)
-        .unstable(unstableTransactionPoolOptions.toDomainObject())
-        .saveFile((dataPath.resolve(stableTxPoolOption.getSaveFile().getPath()).toFile()))
+    return unstableTransactionPoolOptions
+        .toDomainObject()
+        .enableSaveRestore(txPoolOptionGroup.saveRestoreEnabled)
+        .disableLocalTransactions(txPoolOptionGroup.disableLocalTxs)
+        .txPoolLimitByAccountPercentage(txPoolOptionGroup.txPoolLimitByAccountPercentage)
+        .txPoolMaxSize(txPoolOptionGroup.txPoolMaxSize)
+        .pendingTxRetentionPeriod(txPoolOptionGroup.pendingTxRetentionPeriod)
+        .priceBump(Percentage.fromInt(txPoolOptionGroup.priceBump))
+        .txFeeCap(txFeeCap)
+        .saveFile(dataPath.resolve(txPoolOptionGroup.saveFile.getPath()).toFile())
         .build();
   }
 
+  /**
+   * Return the file where to save txpool content if the relative option is enabled.
+   *
+   * @return the save file
+   */
+  public File getSaveFile() {
+    return txPoolOptionGroup.saveFile;
+  }
+
   private boolean isPruningEnabled() {
     return pruningEnabled;
   }
@@ -3408,8 +3489,7 @@ private void setMergeConfigOptions() {
         getActualGenesisConfigOptions().getTerminalTotalDifficulty().isPresent());
   }
 
-  /** Set ignorable segments in RocksDB Storage Provider plugin. */
-  public void setIgnorableStorageSegments() {
+  private void setIgnorableStorageSegments() {
     if (!unstableChainPruningOptions.getChainDataPruningEnabled()) {
       rocksDBPlugin.addIgnorableSegmentIdentifier(KeyValueSegmentIdentifier.CHAIN_PRUNER_STATE);
     }
@@ -3532,7 +3612,9 @@ private String generateConfigurationOverview() {
       builder.setHighSpecEnabled();
     }
 
-    builder.setTxPoolImplementation(buildTransactionPoolConfiguration().getTxPoolImplementation());
+    if (buildTransactionPoolConfiguration().getLayeredTxPoolEnabled()) {
+      builder.setLayeredTxPoolEnabled();
+    }
 
     return builder.build();
   }

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -15,7 +15,6 @@
 package org.hyperledger.besu.cli;
 
 import org.hyperledger.besu.BesuInfo;
-import org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration;
 import org.hyperledger.besu.util.log.FramedLogMessage;
 import org.hyperledger.besu.util.platform.PlatformDetector;
 
@@ -48,7 +47,7 @@ public class ConfigurationOverviewBuilder {
   private Collection<String> engineApis;
   private String engineJwtFilePath;
   private boolean isHighSpec = false;
-  private TransactionPoolConfiguration.Implementation txPoolImplementation;
+  private boolean isLayeredTxPool = false;
   private Map<String, String> environment;
 
   /**
@@ -168,14 +167,12 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() {
   }
 
   /**
-   * Sets the txpool implementation in use.
+   * Sets experimental layered txpool enabled.
    *
-   * @param implementation the txpool implementation
    * @return the builder
    */
-  public ConfigurationOverviewBuilder setTxPoolImplementation(
-      final TransactionPoolConfiguration.Implementation implementation) {
-    txPoolImplementation = implementation;
+  public ConfigurationOverviewBuilder setLayeredTxPoolEnabled() {
+    isLayeredTxPool = true;
     return this;
   }
 
@@ -254,7 +251,9 @@ public String build() {
       lines.add("Experimental high spec configuration enabled");
     }
 
-    lines.add("Using " + txPoolImplementation + " transaction pool implementation");
+    if (isLayeredTxPool) {
+      lines.add("Experimental layered transaction pool configuration enabled");
+    }
 
     lines.add("");
     lines.add("Host:");