test: add browser benches (#570)

* refactor: modularize server-fixture

* Update crates/server-fixture/server/Cargo.toml

add newline

Co-authored-by: sinu.eth <65924192+sinui0@users.noreply.github.com>

* test: add browser benches

* fix deps

* ci: run ci workflow for all pull requests (#571)

* misc fixes

* fix clippy

* don't log a non-critical error to stderr

* use incognito (mitigates random hangs)

* add notes

* distinguish prover kind when plotting

---------

Co-authored-by: sinu.eth <65924192+sinui0@users.noreply.github.com>
Co-authored-by: Hendrik Eeckhaut <hendrik@eeckhaut.org>
Co-authored-by: Ubuntu <ubuntu@ip-10-35-1-164.eu-central-1.compute.internal>

.dockerignore

@@ -0,0 +1,2 @@
+/target
+/.git

.github/workflows/bench-browser.yml

@@ -0,0 +1,27 @@
+name: Run Benchmarks In The Browser
+on:
+  # manual trigger
+  workflow_dispatch:
+
+jobs:
+  run-benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Build Docker Image
+      run: |
+        docker build -t tlsn-bench . -f ./crates/benches/binary/benches.Dockerfile --build-arg BENCH_TYPE=browser
+
+    - name: Run Benchmarks
+      run: |
+        docker run --privileged -v ${{ github.workspace }}/crates/benches/binary:/benches tlsn-bench
+
+    - name: Upload runtime_vs_latency.html
+      uses: actions/upload-artifact@v4
+      with:
+        name: benchmark_graphs
+        path: |
+          ./crates/benches/binary/runtime_vs_latency.html
+          ./crates/benches/binary/runtime_vs_bandwidth.html

.github/workflows/bench.yml

@@ -12,16 +12,16 @@ jobs:
 
     - name: Build Docker Image
       run: |
-        docker build -t tlsn-bench . -f ./crates/benches/benches.Dockerfile
+        docker build -t tlsn-bench . -f ./crates/benches/binary/benches.Dockerfile --build-arg BENCH_TYPE=native
 
     - name: Run Benchmarks
       run: |
-        docker run --privileged -v ${{ github.workspace }}/crates/benches/:/benches tlsn-bench
+        docker run --privileged -v ${{ github.workspace }}/crates/benches/binary:/benches tlsn-bench
 
     - name: Upload runtime_vs_latency.html
       uses: actions/upload-artifact@v4
       with:
         name: benchmark_graphs
         path: |
-          ./crates/benches/runtime_vs_latency.html
-          ./crates/benches/runtime_vs_bandwidth.html
+          ./crates/benches/binary/runtime_vs_latency.html
+          ./crates/benches/binary/runtime_vs_bandwidth.html

.github/workflows/ci.yml

@@ -7,8 +7,6 @@ on:
     tags:
       - "[v]?[0-9]+.[0-9]+.[0-9]+*"
   pull_request:
-    branches:
-      - dev
 
 env:
   CARGO_TERM_COLOR: always

Cargo.toml

@@ -1,6 +1,10 @@
 [workspace]
 members = [
-    "crates/benches",
+    "crates/benches/binary",
+    "crates/benches/browser/core",
+    "crates/benches/browser/native",
+    "crates/benches/browser/wasm",
+    "crates/benches/library",
     "crates/common",
     "crates/components/aead",
     "crates/components/block-cipher",
@@ -41,6 +45,9 @@ notary-client = { path = "crates/notary/client" }
 notary-server = { path = "crates/notary/server" }
 tls-server-fixture = { path = "crates/tls/server-fixture" }
 tlsn-aead = { path = "crates/components/aead" }
+tlsn-benches-browser-core = { path = "crates/benches/browser/core" }
+tlsn-benches-browser-native = { path = "crates/benches/browser/native" }
+tlsn-benches-library = { path = "crates/benches/library" }
 tlsn-block-cipher = { path = "crates/components/block-cipher" }
 tlsn-common = { path = "crates/common" }
 tlsn-core = { path = "crates/core" }

crates/benches/Cargo.toml → crates/benches/binary/Cargo.toml

@@ -2,32 +2,33 @@
 edition = "2021"
 name = "tlsn-benches"
 publish = false
-version = "0.0.0"
+version = "0.1.0"
+
+[features]
+default = []
+browser-bench = ["tlsn-benches-browser-native"]
 
 [dependencies]
-anyhow = { workspace = true }
-charming = { version = "0.3.1", features = ["ssr"] }
-csv = "1.3.0"
-futures = { workspace = true }
-serde = { workspace = true }
+tlsn-benches-library = { workspace = true }
+tlsn-benches-browser-native = { workspace = true, optional = true}
 tlsn-common = { workspace = true }
 tlsn-core = { workspace = true }
 tlsn-prover = { workspace = true }
 tlsn-server-fixture = { workspace = true }
 tlsn-server-fixture-certs = { workspace = true }
 tlsn-tls-core = { workspace = true }
 tlsn-verifier = { workspace = true }
-tokio = { workspace = true, features = [
-  "rt",
-  "rt-multi-thread",
-  "macros",
-  "net",
-  "io-std",
-  "fs",
-] }
+
+anyhow = { workspace = true }
+async-trait = { workspace = true }
+charming = {version = "0.3.1", features = ["ssr"]}
+csv = "1.3.0"
+env_logger = { version = "0.6.0", default-features = false }
+serde = { workspace = true }
+tokio = { workspace = true, features = ["rt", "rt-multi-thread", "macros", "net", "io-std"]}
 tokio-util = { workspace = true }
 toml = "0.8.11"
-tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tracing-subscriber = {workspace = true, features = ["env-filter"]}
 
 [[bin]]
 name = "bench"
@@ -43,4 +44,4 @@ path = "bin/verifier.rs"
 
 [[bin]]
 name = "plot"
-path = "bin/plot.rs"
+path = "bin/plot.rs"

crates/benches/README.md → crates/benches/binary/README.md

@@ -19,13 +19,31 @@ sudo apt-get install iproute2 -y
 
 Running the benches requires root privileges because they will set up virtual interfaces. The script is designed to fully clean up when the benches are done, but run them at your own risk.
 
-Make sure you're in the `crates/benches/` directory, build the binaries then run the script:
+#### Native benches
+
+Make sure you're in the `crates/benches/` directory, build the binaries, and then run the script:
 
 ```sh
+cd binary
 cargo build --release
 sudo ./bench.sh
 ```
 
+#### Browser benches
+
+(Note, we recommend running browser benches inside a docker container (see docker.md) to avoid
+facing incompatibility issues observed in the latest versions of Chrome.)
+
+With a Chrome browser installed on your system, make sure you're in the `crates/benches/` 
+directory, build the wasm module, build the binaries, and then run the script:
+```sh
+cd browser/wasm
+rustup run nightly wasm-pack build --release --target web
+cd ../../binary
+cargo build --release --features browser-bench
+sudo ./bench.sh
+```
+
 ## Metrics
 
 After you run the benches you will see a `metrics.csv` file in the working directory. It will be owned by `root`, so you probably want to run

crates/benches/bench.sh → crates/benches/binary/bench.sh

@@ -7,7 +7,7 @@ if [ "$EUID" -ne 0 ]; then
 fi
 
 # Run the benchmark binary
-../../target/release/bench
+../../../target/release/bench
 
 # Plot the results
-../../target/release/plot metrics.csv
+../../../target/release/plot metrics.csv

crates/benches/bench.toml → crates/benches/binary/bench.toml

@@ -35,5 +35,7 @@ upload-delay = 25
 download = 250
 download-delay = 25
 upload-size = 1024
-download-size = [1024, 4096, 16384, 65536]
+# Setting download-size higher than 45000 will cause a `Maximum call stack size exceeded`
+# error in the browser.
+download-size = [1024, 4096, 16384, 45000]
 defer-decryption = true

crates/benches/binary/benches.Dockerfile

@@ -0,0 +1,46 @@
+FROM rust AS builder
+WORKDIR /usr/src/tlsn
+COPY . .
+
+ARG BENCH_TYPE=native
+
+RUN \
+  if [ "$BENCH_TYPE" = "browser" ]; then \
+    # ring's build script needs clang.
+    apt update && apt install -y clang; \
+    rustup install nightly; \
+    rustup component add rust-src --toolchain nightly; \
+    cargo install wasm-pack; \
+    cd crates/benches/browser/wasm; \
+    rustup run nightly wasm-pack build --release --target web; \
+    cd ../../binary; \
+    cargo build --release --features browser-bench; \
+  else \
+    cd crates/benches/binary; \
+    cargo build --release; \
+  fi
+
+FROM debian:latest
+
+ARG BENCH_TYPE=native
+
+RUN apt update && apt upgrade -y && apt install -y --no-install-recommends \
+  iproute2 \
+  sudo
+
+RUN \
+  if [ "$BENCH_TYPE" = "browser" ]; then \
+    # Using Chromium since Chrome for Linux is not available on ARM.
+    apt install -y chromium; \
+  fi
+
+RUN apt clean && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder ["/usr/src/tlsn/target/release/bench", "/usr/src/tlsn/target/release/prover", "/usr/src/tlsn/target/release/verifier", "/usr/src/tlsn/target/release/plot", "/usr/local/bin/"]
+
+ENV PROVER_PATH="/usr/local/bin/prover"
+ENV VERIFIER_PATH="/usr/local/bin/verifier"
+
+VOLUME [ "/benches" ]
+WORKDIR "/benches"
+CMD ["/bin/bash", "-c", "bench && plot /benches/metrics.csv && cat /benches/metrics.csv"]

crates/benches/bin/bench.rs → crates/benches/binary/bin/bench.rs

@@ -1,19 +1,19 @@
-use std::process::Command;
+use std::{process::Command, thread, time::Duration};
 
 use tlsn_benches::{clean_up, set_up};
 
 fn main() {
-    let prover_path =
-        std::env::var("PROVER_PATH").unwrap_or_else(|_| "../../target/release/prover".to_string());
+    let prover_path = std::env::var("PROVER_PATH")
+        .unwrap_or_else(|_| "../../../target/release/prover".to_string());
     let verifier_path = std::env::var("VERIFIER_PATH")
-        .unwrap_or_else(|_| "../../target/release/verifier".to_string());
+        .unwrap_or_else(|_| "../../../target/release/verifier".to_string());
 
     if let Err(e) = set_up() {
         println!("Error setting up: {}", e);
         clean_up();
     }
 
-    // Run prover and verifier binaries in parallel
+    // Run prover and verifier binaries in parallel.
     let Ok(mut verifier) = Command::new("ip")
         .arg("netns")
         .arg("exec")
@@ -25,6 +25,10 @@ fn main() {
         return clean_up();
     };
 
+    // Allow the verifier some time to start listening before the prover attempts to
+    // connect.
+    thread::sleep(Duration::from_secs(1));
+
     let Ok(mut prover) = Command::new("ip")
         .arg("netns")
         .arg("exec")
@@ -36,7 +40,7 @@ fn main() {
         return clean_up();
     };
 
-    // Wait for both to finish
+    // Wait for both to finish.
     _ = prover.wait();
     _ = verifier.wait();
 

crates/benches/bin/plot.rs → crates/benches/binary/bin/plot.rs

@@ -1,3 +1,5 @@
+use tlsn_benches::metrics::Metrics;
+
 use charming::{
     component::{
         Axis, DataView, Feature, Legend, Restore, SaveAsImage, Title, Toolbox, ToolboxDataZoom,
@@ -7,7 +9,7 @@ use charming::{
     theme::Theme,
     Chart, HtmlRenderer,
 };
-use tlsn_benches::metrics::Metrics;
+use csv::Reader;
 
 const THEME: Theme = Theme::Default;
 
@@ -16,12 +18,12 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .nth(1)
         .expect("Usage: plot <path_to_csv_file>");
 
-    let mut rdr = csv::Reader::from_path(csv_file)?;
+    let mut rdr = Reader::from_path(csv_file)?;
 
-    // Prepare data for plotting
+    // Prepare data for plotting.
     let all_data: Vec<Metrics> = rdr
         .deserialize::<Metrics>()
-        .collect::<Result<Vec<_>, _>>()?; // Attempt to collect all results, return an error if any fail
+        .collect::<Result<Vec<_>, _>>()?; // Attempt to collect all results, return an error if any fail.
 
     let _chart = runtime_vs_latency(&all_data)?;
     let _chart = runtime_vs_bandwidth(&all_data)?;
@@ -32,6 +34,11 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
 fn runtime_vs_latency(all_data: &[Metrics]) -> Result<Chart, Box<dyn std::error::Error>> {
     const TITLE: &str = "Runtime vs Latency";
 
+    let prover_kind: String = all_data
+        .first()
+        .map(|s| s.kind.clone().into())
+        .unwrap_or_default();
+
     let data: Vec<Vec<f32>> = all_data
         .iter()
         .filter(|record| record.name == "latency")
@@ -43,7 +50,11 @@ fn runtime_vs_latency(all_data: &[Metrics]) -> Result<Chart, Box<dyn std::error:
 
     // https://github.com/yuankunzhang/charming
     let chart = Chart::new()
-        .title(Title::new().text(TITLE))
+        .title(
+            Title::new()
+                .text(TITLE)
+                .subtext(format!("{} Prover", prover_kind)),
+        )
         .tooltip(Tooltip::new().trigger(Trigger::Axis))
         .legend(Legend::new().orient(Orient::Vertical))
         .toolbox(