feat(hyp-ethereum): gas escalator middleware (#3852)

### Description

Wraps evm providers in a gas escalator middleware as part of the
`build_with_signer` step, so signed txs can have their gas price bumped
if stuck in the mempool.

Gas price is increased every 60s by 12.5%, and is capped at the
`max_fee_per_gas` set in tx overrides.

### Drive-by changes

- changes to our fork of `ethers`:
- removes the `M: Clone` bound, since it's not actually required and
conflicts with the middleware we use
- increases the level of the `escalated` log in the middleware to
`debug` (so we can see it working in prod without enabling `trace`)

### Related issues

- Fixes #2959

### Backward compatibility

Yes

### Testing

e2e

---------

Co-authored-by: Trevor Porter <tkporter4@gmail.com>

rust/Cargo.toml

@@ -196,27 +196,27 @@ cosmwasm-schema = "1.2.7"
 [workspace.dependencies.ethers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-06-27-2"
 
 [workspace.dependencies.ethers-contract]
 features = ["legacy"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-06-27-2"
 
 [workspace.dependencies.ethers-core]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-06-27-2"
 
 [workspace.dependencies.ethers-providers]
 features = []
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-06-27-2"
 
 [workspace.dependencies.ethers-signers]
 features = ["aws"]
 git = "https://github.com/hyperlane-xyz/ethers-rs"
-tag = "2024-04-25"
+tag = "2024-06-27-2"
 
 [patch.crates-io.curve25519-dalek]
 branch = "v3.2.2-relax-zeroize"

rust/chains/hyperlane-ethereum/src/rpc_clients/trait_builder.rs

@@ -3,13 +3,16 @@ use std::sync::Arc;
 use std::time::Duration;
 
 use async_trait::async_trait;
+use ethers::middleware::gas_escalator::{Frequency, GasEscalatorMiddleware, GeometricGasPrice};
 use ethers::middleware::gas_oracle::{
     GasCategory, GasOracle, GasOracleMiddleware, Polygon, ProviderOracle,
 };
 use ethers::prelude::{
     Http, JsonRpcClient, Middleware, NonceManagerMiddleware, Provider, Quorum, QuorumProvider,
     SignerMiddleware, WeightedProvider, Ws, WsClientError,
 };
+use ethers::types::Address;
+use ethers_signers::Signer;
 use hyperlane_core::rpc_clients::FallbackProvider;
 use reqwest::{Client, Url};
 use thiserror::Error;
@@ -210,10 +213,19 @@ pub trait BuildableWithProvider {
         M: Middleware + 'static,
     {
         Ok(if let Some(signer) = signer {
-            let signing_provider = wrap_with_signer(provider, signer)
+            // The signing provider is used for sending txs, which may end up stuck in the mempool due to
+            // gas pricing issues. We first wrap the provider in a signer middleware, to sign any new txs sent by the gas escalator middleware.
+            // We keep nonce manager as the outermost middleware, so that every new tx with a higher gas price reuses the same nonce.
+            let signing_provider = wrap_with_signer(provider, signer.clone())
                 .await
                 .map_err(ChainCommunicationError::from_other)?;
-            self.build_with_provider(signing_provider, conn, locator)
+            let gas_escalator_provider = wrap_with_gas_escalator(signing_provider);
+            let nonce_manager_provider =
+                wrap_with_nonce_manager(gas_escalator_provider, signer.address())
+                    .await
+                    .map_err(ChainCommunicationError::from_other)?;
+
+            self.build_with_provider(nonce_manager_provider, conn, locator)
         } else {
             self.build_with_provider(provider, conn, locator)
         }
@@ -234,15 +246,19 @@ pub trait BuildableWithProvider {
 async fn wrap_with_signer<M: Middleware>(
     provider: M,
     signer: Signers,
-) -> Result<SignerMiddleware<NonceManagerMiddleware<M>, Signers>, M::Error> {
+) -> Result<SignerMiddleware<M, Signers>, M::Error> {
     let provider_chain_id = provider.get_chainid().await?;
     let signer = ethers::signers::Signer::with_chain_id(signer, provider_chain_id.as_u64());
 
-    let address = ethers::prelude::Signer::address(&signer);
-    let provider = NonceManagerMiddleware::new(provider, address);
+    Ok(SignerMiddleware::new(provider, signer))
+}
 
-    let signing_provider = SignerMiddleware::new(provider, signer);
-    Ok(signing_provider)
+async fn wrap_with_nonce_manager<M: Middleware>(
+    provider: M,
+    signer_address: Address,
+) -> Result<NonceManagerMiddleware<M>, M::Error> {
+    let nonce_manager_provider = NonceManagerMiddleware::new(provider, signer_address);
+    Ok(nonce_manager_provider)
 }
 
 fn build_polygon_gas_oracle(chain: ethers_core::types::Chain) -> ChainResult<Box<dyn GasOracle>> {
@@ -274,3 +290,20 @@ where
     };
     Ok(GasOracleMiddleware::new(provider, gas_oracle))
 }
+
+fn wrap_with_gas_escalator<M>(provider: M) -> GasEscalatorMiddleware<M>
+where
+    M: Middleware + 'static,
+{
+    // Increase the gas price by 12.5% every 60 seconds
+    // (These are the default values from ethers doc comments)
+    const COEFFICIENT: f64 = 1.125;
+    const EVERY_SECS: u64 = 60u64;
+    // 550 gwei is the limit we also use for polygon, so we reuse for consistency
+    const MAX_GAS_PRICE: u128 = 550 * 10u128.pow(9);
+    let escalator = GeometricGasPrice::new(COEFFICIENT, EVERY_SECS, MAX_GAS_PRICE.into());
+    // Check the status of sent txs every eth block or so. The alternative is to subscribe to new blocks and check then,
+    // which adds unnecessary load on the provider.
+    const FREQUENCY: Frequency = Frequency::Duration(Duration::from_secs(12).as_millis() as _);
+    GasEscalatorMiddleware::new(provider, escalator, FREQUENCY)
+}

rust/chains/hyperlane-ethereum/src/tx.rs

@@ -44,8 +44,7 @@ where
         .cloned()
         .unwrap_or_else(|| NameOrAddress::Address(Default::default()));
 
-    info!(?to, %data, "Dispatching transaction");
-    // We can set the gas higher here!
+    info!(?to, %data, tx=?tx.tx, "Dispatching transaction");
     let dispatch_fut = tx.send();
     let dispatched = dispatch_fut
         .await?

rust/ethers-prometheus/src/middleware/mod.rs

@@ -227,7 +227,6 @@ impl<M: Middleware> Middleware for PrometheusMiddleware<M> {
     ) -> Result<PendingTransaction<'_, Self::Provider>, Self::Error> {
         let start = Instant::now();
         let tx: TypedTransaction = tx.into();
-
         let chain = {
             let data = self.conf.read().await;
             chain_name(&data.chain).to_owned()

rust/utils/run-locally/src/invariants.rs

@@ -80,9 +80,12 @@ pub fn termination_invariants_met(
     // EDIT: Having had a quick look, it seems like there are some legitimate reverts happening in the confirm step
     // (`Transaction attempting to process message either reverted or was reorged`)
     // in which case more gas expenditure logs than messages are expected.
+    let gas_expenditure_log_count = log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap();
     assert!(
-        log_counts.get(GAS_EXPENDITURE_LOG_MESSAGE).unwrap() >= &total_messages_expected,
-        "Didn't record gas payment for all delivered messages"
+        gas_expenditure_log_count >= &total_messages_expected,
+        "Didn't record gas payment for all delivered messages. Got {} gas payment logs, expected at least {}",
+        gas_expenditure_log_count,
+        total_messages_expected
     );
     // These tests check that we fixed https://github.com/hyperlane-xyz/hyperlane-monorepo/issues/3915, where some logs would not show up
     assert!(