demo native marketplace (#1944)

Adds marketplace demo-native workflow * adds a justfile entry * updates process-compose configuration with marketplace version values * fixes a bug where `max_base_fee` was not taking in `Marketplace` upgrade values * updates `smoke-test-demo` to account for marketplace load-generator * adds genesis file for marketplace --------- Co-authored-by: tbro <tbro@users.noreply.github.com> Co-authored-by: Abdul Basit <45506001+imabdulbasit@users.noreply.github.com>
EspressoSystems · Sep 5, 2024 · 3c0db34 · 3c0db34
1 parent b121de9
commit 3c0db34
Show file tree

Hide file tree

Showing 12 changed files with 228 additions and 65 deletions.
diff --git a/.env b/.env
@@ -105,6 +105,7 @@ ESPRESSO_BUILDER_L1_PROVIDER=${ESPRESSO_SEQUENCER_L1_PROVIDER}
 ESPRESSO_BUILDER_ETH_MNEMONIC=${ESPRESSO_SEQUENCER_ETH_MNEMONIC}
 ESPRESSO_BUILDER_SERVER_PORT=31003
 ESPRESSO_FALLBACK_BUILDER_SERVER_PORT=31004
+ESPRESSO_RESERVE_BUILDER_SERVER_PORT=31005
 ESPRESSO_BUILDER_TX_CHANNEL_CAPACITY=4096
 ESPRESSO_BUILDER_EVENT_CHANNEL_CAPACITY=128
 ESPRESSO_BUILDER_INIT_NODE_COUNT=$ESPRESSO_ORCHESTRATOR_NUM_NODES
@@ -117,6 +118,8 @@ ESPRESSO_BUILDER_GENESIS_FILE=$ESPRESSO_SEQUENCER_GENESIS_FILE
 ESPRESSO_SUBMIT_TRANSACTIONS_DELAY=2s
 ESPRESSO_SUBMIT_TRANSACTIONS_PUBLIC_PORT=24010
 ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT=24020
+ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_FALLBACK_PORT=24030
+ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_RESERVE_PORT=24040
 
 # Benchmarks
 ESPRESSO_BENCH_START_BLOCK=50

diff --git a/.github/workflows/test-demo-native-marketplace.yml b/.github/workflows/test-demo-native-marketplace.yml
@@ -0,0 +1,54 @@
+name: Test Demo Native - Marketplace
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      # YYYYMMDD
+      - "20[0-9][0-9][0-1][0-9][0-3][0-9]*"
+  schedule:
+    - cron: "0 0 * * 1"
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUSTFLAGS: '--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std"'
+  RUST_LOG: info,libp2p=off,node=error
+
+jobs:
+  demo-native-marketplace:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@V27
+
+      - name: Install process-compose
+        run: |
+          nix profile install nixpkgs#process-compose
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+
+      - name: Build
+        run: cargo build --locked --release
+
+      - name: Test Demo Marketplace
+        run: |
+          export MARKETPLACE_SMOKE_TEST=true
+          set -o pipefail
+          scripts/demo-native --tui=false -f process-compose.yaml -f process-compose-mp.yml &
+          timeout -v 600 scripts/smoke-test-demo | sed -e 's/^/smoke-test: /;'
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -61,4 +61,4 @@ jobs:
           export RUSTFLAGS="$RUSTFLAGS --cfg hotshot_example"
           export PATH="$PWD/target/release:$PATH"
           cargo nextest run --locked --release --workspace --all-features --retries 2 --verbose -E '!test(slow_)'
-        timeout-minutes: 5
+        timeout-minutes: 10
diff --git a/builder/src/bin/permissionless-builder.rs b/builder/src/bin/permissionless-builder.rs
@@ -134,6 +134,7 @@ async fn run<V: Versions>(
         build_instance_state::<V>(genesis.chain_config, l1_params, opt.state_peers).unwrap();
 
     let base_fee = genesis.max_base_fee();
+    tracing::info!(?base_fee, "base_fee");
 
     let validated_state = ValidatedState::genesis(&instance_state).0;
 

diff --git a/data/genesis/demo-marketplace.toml b/data/genesis/demo-marketplace.toml
@@ -0,0 +1,29 @@
+base_version = "0.2"
+upgrade_version = "0.3"
+
+[stake_table]
+capacity = 10
+
+[chain_config]
+chain_id = 999999999
+base_fee = '0 wei'
+max_block_size = '1mb'
+fee_recipient = '0x0000000000000000000000000000000000000000'
+fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468'
+
+[header]
+timestamp = "1970-01-01T00:00:00Z"
+
+[[upgrade]]
+version = "0.3"
+start_proposing_view = 5
+stop_proposing_view = 15
+
+[upgrade.marketplace]
+[upgrade.marketplace.chain_config]
+chain_id = 999999999
+max_block_size = '1mb'
+base_fee = '1 wei'
+fee_recipient = "0x0000000000000000000000000000000000000000"
+bid_recipient = "0x0000000000000000000000000000000000000000"
+fee_contract = '0xa15bb66138824a1c7167f5e85b957d04dd34e468'
diff --git a/justfile b/justfile
@@ -11,6 +11,10 @@ demo-native:
     cargo build --release
     scripts/demo-native
 
+demo-native-mp:
+    cargo build --release
+    scripts/demo-native -f process-compose.yaml -f process-compose-mp.yml
+
 demo-native-benchmark:
     cargo build --release --features benchmarking
     scripts/demo-native

diff --git a/marketplace-builder/src/bin/marketplace-builder.rs b/marketplace-builder/src/bin/marketplace-builder.rs
@@ -171,6 +171,7 @@ async fn run<V: Versions>(
         build_instance_state::<V>(genesis.chain_config, l1_params, opt.state_peers).unwrap();
 
     let base_fee = genesis.max_base_fee();
+    tracing::info!(?base_fee, "base_fee");
 
     let validated_state = ValidatedState::genesis(&instance_state).0;
 

diff --git a/marketplace-solver/src/events.rs b/marketplace-solver/src/events.rs
@@ -52,13 +52,13 @@ pub async fn handle_events(
     while let Some(event) = stream.next().await {
         let event = event?;
 
-        tracing::info!("received event {:?}", event.event);
+        tracing::debug!("received event {:?}", event.event);
 
         // TODO ED: Remove this lint later
         #[allow(clippy::single_match)]
         match event.event {
             hotshot::types::EventType::ViewFinished { view_number } => {
-                tracing::info!("received view finished event {view_number:?}")
+                tracing::debug!("received view finished event {view_number:?}")
             }
             _ => (),
         }

diff --git a/process-compose-mp.yml b/process-compose-mp.yml
@@ -0,0 +1,118 @@
+version: "3"
+
+environment:
+  - ESPRESSO_SEQUENCER_GENESIS_FILE=data/genesis/demo-marketplace.toml
+  - ESPRESSO_BUILDER_GENESIS_FILE=data/genesis/demo-marketplace.toml
+processes:
+
+  marketplace-reserve-builder:
+    command: marketplace-builder
+    ports:
+      - "$ESPRESSO_RESERVE_BUILDER_SERVER_PORT:$ESPRESSO_RESERVE_BUILDER_SERVER_PORT"
+    environment:
+      - ESPRESSO_BUILDER_SERVER_PORT=$ESPRESSO_RESERVE_BUILDER_SERVER_PORT
+      - ESPRESSO_MARKETPLACE_BUILDER_IS_RESERVE=true
+      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
+      - ESPRESSO_MARKETPLACE_BUILDER_NAMESPACE=2
+      - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://localhost:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
+      - ESPRESSO_BUILDER_GENESIS_FILE
+    depends_on:
+      sequencer0:
+        condition: process_healthy
+      fund-builder:
+        condition: process_completed
+    readiness_probe:
+      http_get:
+        scheme: http
+        host: localhost
+        port: $ESPRESSO_RESERVE_BUILDER_SERVER_PORT
+        path: /healthcheck
+      failure_threshold: 100
+
+  marketplace-fallback-builder:
+    command: marketplace-builder
+    ports:
+      - "$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT:$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT"
+    environment:
+      - ESPRESSO_BUILDER_SERVER_PORT=$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT
+      - ESPRESSO_MARKETPLACE_BUILDER_IS_RESERVE=false
+      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
+      - ESPRESSO_MARKETPLACE_BUILDER_NAMESPACE=4
+      - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
+      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://localhost:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
+      - ESPRESSO_BUILDER_GENESIS_FILE
+    depends_on:
+      sequencer0:
+        condition: process_healthy
+      fund-builder:
+        condition: process_completed
+    readiness_probe:
+      http_get:
+        scheme: http
+        host: localhost
+        port: $ESPRESSO_FALLBACK_BUILDER_SERVER_PORT
+        path: /healthcheck
+      failure_threshold: 100
+
+  permissionless-builder:
+    environment:
+      - ESPRESSO_MARKETPLACE_BUILDER_NAMESPACE=10
+
+  submit-transactions-public:
+    command: submit-transactions
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MIN_NAMESPACE=9
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MAX_NAMESPACE=10
+
+  submit-transactions-private:
+    disabled: true
+
+  submit-transactions-private-fallback:
+    command: submit-transactions
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_FALLBACK_PORT
+      - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=http://localhost:$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT/txn_submit
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MIN_NAMESPACE=1
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MAX_NAMESPACE=3
+    depends_on:
+      sequencer0:
+        condition: process_healthy
+      marketplace-fallback-builder:
+        condition: process_healthy
+    readiness_probe:
+      http_get:
+        scheme: http
+        host: localhost
+        port: $ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_FALLBACK_PORT
+        path: /healthcheck
+      failure_threshold: 100
+
+  submit-transactions-private-reserve:
+    command: submit-transactions
+    environment:
+      - ESPRESSO_SUBMIT_TRANSACTIONS_PORT=$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_RESERVE_PORT
+      - ESPRESSO_SUBMIT_TRANSACTIONS_SUBMIT_URL=http://localhost:$ESPRESSO_RESERVE_BUILDER_SERVER_PORT/txn_submit
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MIN_NAMESPACE=4
+      - ESPRESSO_SUBMIT_TRANSACTIONS_MAX_NAMESPACE=6
+    depends_on:
+      sequencer0:
+        condition: process_healthy
+      marketplace-reserve-builder:
+        condition: process_healthy
+    readiness_probe:
+      http_get:
+        scheme: http
+        host: localhost
+        port: $ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_RESERVE_PORT
+        path: /healthcheck
+      failure_threshold: 100
+
+  dev-rollup:
+    command:
+      dev-rollup register --ns 1; dev-rollup register --ns 2; dev-rollup register --ns 3
+    environment:
+      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
+    depends_on:
+      marketplace-solver:
+        condition: process_healthy
diff --git a/process-compose.yaml b/process-compose.yaml
@@ -490,57 +490,6 @@ processes:
         path: /healthcheck
       failure_threshold: 100
 
-  marketplace-reserve-builder:
-    disabled: true
-    command: marketplace-builder
-    ports:
-      - "$ESPRESSO_BUILDER_SERVER_PORT:$ESPRESSO_BUILDER_SERVER_PORT"
-    environment:
-      - ESPRESSO_MARKETPLACE_BUILDER_IS_RESERVE=true
-      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
-      - ESPRESSO_MARKETPLACE_BUILDER_NAMESPACE=10000
-      - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
-      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://localhost:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
-      - ESPRESSO_BUILDER_GENESIS_FILE
-    depends_on:
-      sequencer0:
-        condition: process_healthy
-      fund-builder:
-        condition: process_completed
-    readiness_probe:
-      http_get:
-        scheme: http
-        host: localhost
-        port: $ESPRESSO_BUILDER_SERVER_PORT
-        path: /healthcheck
-      failure_threshold: 100
-
-  marketplace-fallback-builder:
-    disabled: true
-    command: marketplace-builder
-    ports:
-      - "$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT:$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT"
-    environment:
-      - ESPRESSO_BUILDER_SERVER_PORT=$ESPRESSO_FALLBACK_BUILDER_SERVER_PORT
-      - ESPRESSO_MARKETPLACE_BUILDER_IS_RESERVE=false
-      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
-      - ESPRESSO_MARKETPLACE_BUILDER_NAMESPACE=10000
-      - ESPRESSO_SEQUENCER_STATE_PEERS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT
-      - ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_URL=http://localhost:$ESPRESSO_SEQUENCER_HOTSHOT_EVENT_STREAMING_API_PORT
-      - ESPRESSO_BUILDER_GENESIS_FILE
-    depends_on:
-      sequencer0:
-        condition: process_healthy
-      fund-builder:
-        condition: process_completed
-    readiness_probe:
-      http_get:
-        scheme: http
-        host: localhost
-        port: $ESPRESSO_FALLBACK_BUILDER_SERVER_PORT
-        path: /healthcheck
-      failure_threshold: 100
-
   permissionless-builder:
     command: permissionless-builder
     ports:
@@ -663,11 +612,3 @@ processes:
       sequencer1:
         condition: process_healthy
 
-  dev-rollup:
-    command:
-      dev-rollup register --ns 1; dev-rollup register --ns 2; dev-rollup register --ns 3
-    environment:
-      - ESPRESSO_MARKETPLACE_SOLVER_API_URL=http://localhost:$ESPRESSO_MARKETPLACE_SOLVER_API_PORT
-    depends_on:
-      marketplace-solver:
-        condition: process_healthy
diff --git a/scripts/smoke-test-demo b/scripts/smoke-test-demo
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -eE
 
+PATH="$PWD/target/release:$PATH"
+
 # Duplicate some debugging information to stdout in case the process exits
 trap 'error_handler "${LINENO}" "$BASH_COMMAND"' ERR INT
 error_handler() {
@@ -12,9 +14,14 @@ error_handler() {
 # load env vars in .env
 set -a; source .env; set +a;
 
+if [ "$MARKETPLACE_SMOKE_TEST" = true ] ; then
+    LOAD_GENERATOR=http://localhost:$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_FALLBACK_PORT
+else
+    LOAD_GENERATOR=http://localhost:$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT
+fi
+
 L1_PROVIDER=http://localhost:$ESPRESSO_SEQUENCER_L1_PORT
 SEQUENCER_API=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT
-LOAD_GENERATOR=http://localhost:$ESPRESSO_SUBMIT_TRANSACTIONS_PRIVATE_PORT
 SEQUENCER_BLOCKS_TIMEOUT=120
 
 # usage: wait_for timeout_in_secs name URL

diff --git a/sequencer/src/genesis.rs b/sequencer/src/genesis.rs
@@ -60,8 +60,13 @@ impl Genesis {
         let upgrades: Vec<&Upgrade> = self.upgrades.values().collect();
 
         for upgrade in upgrades {
-            if let UpgradeType::Fee { chain_config } = upgrade.upgrade_type {
-                base_fee = std::cmp::max(chain_config.base_fee, base_fee);
+            match upgrade.upgrade_type {
+                UpgradeType::Fee { chain_config } => {
+                    base_fee = std::cmp::max(chain_config.base_fee, base_fee);
+                }
+                UpgradeType::Marketplace { chain_config } => {
+                    base_fee = std::cmp::max(chain_config.base_fee, base_fee);
+                }
             }
         }