Merge pull request #151 from ewasm/scout-wamr

add scout_wamr.c to scout engines
ewasm · Aug 14, 2020 · f1ed847 · f1ed847
2 parents 5d93841 + 8002b09
commit f1ed847
Show file tree

Hide file tree

Showing 39 changed files with 563 additions and 492 deletions.
diff --git a/benchmark_results_data/scout_bignum_benchmarks.csv b/benchmark_results_data/scout_bignum_benchmarks.csv
diff --git a/images/bignums-bls12-pairings-fizzy-vs-wabt-exec-startup-time.png b/images/bignums-bls12-pairings-fizzy-vs-wabt-exec-startup-time.png
diff --git a/images/bignums-bls12-pairings-host-func-variations-exec-startup-time.png b/images/bignums-bls12-pairings-host-func-variations-exec-startup-time.png
diff --git a/images/bignums-bls12-pairings-host-func-variations-total-time.png b/images/bignums-bls12-pairings-host-func-variations-total-time.png
diff --git a/images/bignums-bls12-pairings-rust-vs-wasm-exec-startup-time.png b/images/bignums-bls12-pairings-rust-vs-wasm-exec-startup-time.png
diff --git a/images/bignums-bls12-pairings-rust-vs-wasm-total-time.png b/images/bignums-bls12-pairings-rust-vs-wasm-total-time.png
diff --git a/images/bignums-bls12-pairings-wabt-vs-native-exec-startup-time.png b/images/bignums-bls12-pairings-wabt-vs-native-exec-startup-time.png
diff --git a/images/bignums-bls12-synth-loop-fizzy-vs-wabt-exec-startup-time.png b/images/bignums-bls12-synth-loop-fizzy-vs-wabt-exec-startup-time.png
diff --git a/images/bignums-rollup-rs-bn128-pairings-all-combos.png b/images/bignums-rollup-rs-bn128-pairings-all-combos.png
diff --git a/images/bignums-rollup-rs-bn128-pairings-superops.png b/images/bignums-rollup-rs-bn128-pairings-superops.png
diff --git a/images/bignums-rollup-rs-bn128-pairings-wabt-with-bignums.png b/images/bignums-rollup-rs-bn128-pairings-wabt-with-bignums.png
diff --git a/images/bignums-rollup-rs-bn128-pairings-with-vs-without-bignums.png b/images/bignums-rollup-rs-bn128-pairings-with-vs-without-bignums.png
diff --git a/images/bignums-websnark-bn128-pairings-fasthost.png b/images/bignums-websnark-bn128-pairings-fasthost.png
diff --git a/images/bignums-websnark-bn128-pairings-fastmont.png b/images/bignums-websnark-bn128-pairings-fastmont.png
diff --git a/images/bignums-websnark-bn128-pairings-startup-and-execution.png b/images/bignums-websnark-bn128-pairings-startup-and-execution.png
diff --git a/images/bignums-websnark-bn128-pairings-superops.png b/images/bignums-websnark-bn128-pairings-superops.png
diff --git a/images/bignums-websnark-bn128-pairings-v8-with-bignums.png b/images/bignums-websnark-bn128-pairings-v8-with-bignums.png
diff --git a/images/bignums-zkrollup-rust-vs-websnark-bn128-pairings.png b/images/bignums-zkrollup-rust-vs-websnark-bn128-pairings.png
diff --git a/images/evm384-bls12-pairings-synth-loop-rust-wasm.png b/images/evm384-bls12-pairings-synth-loop-rust-wasm.png
diff --git a/images/evm384-bls12-synth-loop-vs-native-rust.png b/images/evm384-bls12-synth-loop-vs-native-rust.png
diff --git a/images/evm384-bls12-synth-loop-vs-pairings.png b/images/evm384-bls12-synth-loop-vs-pairings.png
diff --git a/images/evm384-bls12-synth-loop-wasm.png b/images/evm384-bls12-synth-loop-wasm.png
diff --git a/images/evm384-wasm-native-pairings-and-synth-loop.png b/images/evm384-wasm-native-pairings-and-synth-loop.png
diff --git a/images/evm384-wasm-native-pairings-vs-adjusted-synth-loop.png b/images/evm384-wasm-native-pairings-vs-adjusted-synth-loop.png
diff --git a/images/scout-biturbo-token-all-engines.png b/images/scout-biturbo-token-all-engines.png
diff --git a/images/scout-biturbo-token-compilers-vs-interpreters.png b/images/scout-biturbo-token-compilers-vs-interpreters.png
diff --git a/images/scout-daiquiri-all-engines.png b/images/scout-daiquiri-all-engines.png
diff --git a/images/scout-daiquiri-v8-vs-wabt.png b/images/scout-daiquiri-v8-vs-wabt.png
diff --git a/images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-all-engines.png b/images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-all-engines.png
diff --git a/images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-v8.png b/images/scout-ecpairing-zkrollup-rust-wasm-bn128-two-pairings-v8.png
diff --git a/images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-all-engines.png b/images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-all-engines.png
diff --git a/images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-v8.png b/images/scout-ecpairing-zkrollup-websnark-bn128-two-pairings-v8.png
diff --git a/images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-all-engines.png b/images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-all-engines.png
diff --git a/images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-wabt-vs-v8.png b/images/scout-ecrecover-websnark-secp256k1-verify-72-sigs-wabt-vs-v8.png
diff --git a/images/websnark-vs-native-bn128-two-pairings-v8.png b/images/websnark-vs-native-bn128-two-pairings-v8.png
diff --git a/images/websnark-vs-rust-wasm-native-zkrollup-bn128-pairings.png b/images/websnark-vs-rust-wasm-native-zkrollup-bn128-pairings.png
diff --git a/notebooks/wasm-engines.ipynb b/notebooks/wasm-engines.ipynb
@@ -5956,6 +5956,7 @@
    "outputs": [],
    "source": [
     "\n",
+    "# test to make  sure that scout wasm3 data is present\n",
     "test_scout_wasm3_plot = plotOneTestColoredTicks(df_scout_data[df_scout_data['engine'].isin(['wasm3-with-bignums'])],)\n"
    ]
   },
@@ -5964,7 +5965,11 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "\n",
+    "# test to make sure that scout_wamr.c data is present\n",
+    "test_scout_wasm3_plot = plotOneTestColoredTicks(df_scout_data[df_scout_data['engine'].isin(['wasm3-with-bignums'])],)\n"
+   ]
   },
   {
    "cell_type": "code",

diff --git a/scout-engines/Dockerfile b/scout-engines/Dockerfile
@@ -184,9 +184,18 @@ RUN git clone --single-branch --branch dev https://github.com/jwasinger/eip1962-
 RUN cd eip1962-bls12-rs-native && git submodule update --init
 RUN cd eip1962-bls12-rs-native && cargo build --release
 
+WORKDIR /engines
+
+# install scout-wamr
+RUN git clone https://github.com/ewasm-benchmarking/scout_wamr.c -b bls12-test-cases && \
+	cd scout_wamr.c && \
+	git submodule update --init && \
+	mkdir build && \
+	cd build && \
+	cmake .. && \
+	make -j4
 
 # install fizzy
-WORKDIR /engines
 
 # fizzy for bls12 using host funcs
 RUN git clone --single-branch --branch v0.3.0-ewasm-bench-bls12 https://github.com/ewasm-benchmarking/fizzy.git fizzy-bls12-hostfuncs && \

diff --git a/scout-engines/scout_bignum_bench.py b/scout-engines/scout_bignum_bench.py
@@ -43,6 +43,20 @@
   }
 ]
 
+WAMR_BENCH_INFOS = [
+  {
+    'bench_name': 'bls12-wasmsnark-synth-loop',
+    'engine_path': '/engines/scout_wamr.c/build/scout.exec',
+    'engine_name': 'wamr-with-bignums',
+    'scout_yml_path': '/engines/scout_wamr.c/bls12-synth-loop.yaml',
+  },
+  {
+    'bench_name': 'bls12-wasmsnark-two-pairings-standalone',
+    'engine_path': '/engines/scout_wamr.c/build/scout.exec',
+    'engine_name': 'wamr-with-bignums',
+    'scout_yml_path': '/engines/scout_wamr.c/bls12-two-pairings-standalone.yaml',
+  }
+]
 
 FIZZY_BENCH_INFOS = [
   {
@@ -641,8 +655,25 @@ def do_v8_bench(scoutts_cmd, scoutts_working_dir):
     exec_time = exec_time_seconds + exec_time_milliseconds
     return { 'exec_time': exec_time.total_seconds(), 'parse_time': parse_time.total_seconds() }
 
+def do_wamr_bench(wamr_bench_info):
+    wamr_cmd = [wamr_bench_info['engine_path'], wamr_bench_info['scout_yml_path']]
+    stdoutlines = []
+    wamr_working_dir = os.path.dirname(wamr_bench_info['scout_yml_path'])
 
+    with subprocess.Popen(wamr_cmd, cwd=wamr_working_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p:
+        for line in p.stdout: # b'\n'-separated lines
+            print(line, end='')
+            stdoutlines.append(line)  # pass bytes as is
+        p.wait()
 
+    parse_time_regex = 'startup time: (.*)\n'
+    execution_time_regex = 'execution time: (.*)\n'
+    parse_time_match = re.search(parse_time_regex, stdoutlines[0])[1]
+    execution_time_match = re.search(execution_time_regex, stdoutlines[2])[1]
+    return {'engine': wamr_bench_info['engine_name'],
+            'bench_name': wamr_bench_info['bench_name'],
+            'parse_time': float(parse_time_match),
+            'exec_time': float(execution_time_match)}
 
 """
 root@1152c9d9157f:/engines/scout.ts-bn128# /engines/scout-cpp-bn128/build/scout.exec bn128pairing_bignums.yaml
@@ -994,6 +1025,12 @@ def main():
             scout_benchmarks.append(wasm3_record)
             print("got fizzy result:", wasm3_record)
 
+    # run 10 iterations of scout_wamr.c
+
+    for i in range(10):
+        for bench_to_run in WAMR_BENCH_INFOS:
+            scout_benchmarks.append(do_wamr_bench(bench_to_run))
+
     # run 10 iterations of rust-native
     for i in range(0, 10):
         print("\ndoing rust-native bench run i=",i)