Merge pull request #513 from miles170/migrate-to-state-test

Migrate benchmarks to the JSON State Test format

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "evmc"]
 	path = evmc
 	url = https://github.com/ethereum/evmc
+[submodule "evm-benchmarks"]
+	path = test/evm-benchmarks
+	url = https://github.com/ipsilon/evm-benchmarks

test/bench/CMakeLists.txt

@@ -4,7 +4,7 @@
 
 add_executable(evmone-bench)
 target_include_directories(evmone-bench PRIVATE ${evmone_private_include_dir})
-target_link_libraries(evmone-bench PRIVATE evmone testutils evmc::loader benchmark::benchmark)
+target_link_libraries(evmone-bench PRIVATE evmone testutils evmone::statetestutils evmc::loader benchmark::benchmark)
 target_sources(
     evmone-bench PRIVATE
     bench.cpp
@@ -15,9 +15,10 @@ target_sources(
 # Tests
 
 set(PREFIX evmone/bench)
+set(BENCHMARK_SUITE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../evm-benchmarks/benchmarks)
 
 # Check if DIR argument works.
-add_test(NAME ${PREFIX}/dir COMMAND evmone-bench ${CMAKE_CURRENT_SOURCE_DIR}/../benchmarks --benchmark_list_tests)
+add_test(NAME ${PREFIX}/dir COMMAND evmone-bench ${BENCHMARK_SUITE_DIR} --benchmark_list_tests)
 set_tests_properties(${PREFIX}/dir PROPERTIES PASS_REGULAR_EXPRESSION "total/synth")
 
 # Omitting DIR is fine.
@@ -29,7 +30,6 @@ add_test(NAME ${PREFIX}/dirname_empty COMMAND evmone-bench "" --benchmark_list_t
 set_tests_properties(${PREFIX}/dirname_empty PROPERTIES PASS_REGULAR_EXPRESSION "total/synth")
 
 # Run all benchmark cases split into groups to check if none of them crashes.
-set(BENCHMARK_SUITE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../benchmarks)
 add_test(NAME ${PREFIX}/synth COMMAND evmone-bench --benchmark_min_time=0 --benchmark_filter=synth)
 add_test(NAME ${PREFIX}/micro COMMAND evmone-bench --benchmark_min_time=0 --benchmark_filter=micro ${BENCHMARK_SUITE_DIR})
 add_test(NAME ${PREFIX}/main/b COMMAND evmone-bench --benchmark_min_time=0 --benchmark_filter=main/[b] ${BENCHMARK_SUITE_DIR})

test/bench/bench.cpp

@@ -2,6 +2,7 @@
 // Copyright 2019 The evmone Authors.
 // SPDX-License-Identifier: Apache-2.0
 
+#include "../statetest/statetest.hpp"
 #include "helpers.hpp"
 #include "synthetic_benchmarks.hpp"
 #include <benchmark/benchmark.h>
@@ -31,7 +32,7 @@ struct BenchmarkCase
         bytes input;
         bytes expected_output;
 
-        Input(std::string _name, bytes _input, bytes _expected_output) noexcept
+        Input(std::string _name, bytes _input, bytes _expected_output = {}) noexcept
           : name{std::move(_name)},
             input{std::move(_input)},
             expected_output{std::move(_expected_output)}
@@ -41,76 +42,31 @@ struct BenchmarkCase
     std::string name;
     bytes code;
     std::vector<Input> inputs;
-
-    /// Create a benchmark case without input.
-    BenchmarkCase(std::string _name, bytes _code) noexcept
-      : name{std::move(_name)}, code{std::move(_code)}
-    {}
 };
 
-
-constexpr auto runtime_code_extension = ".bin-runtime";
-constexpr auto inputs_extension = ".inputs";
-
 /// Loads the benchmark case's inputs from the inputs file at the given path.
-std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
+std::vector<BenchmarkCase::Input> load_inputs(const StateTransitionTest& state_test)
 {
-    enum class state
-    {
-        name,
-        input,
-        expected_output
-    };
-
-    auto inputs_file = std::ifstream{path};
-
     std::vector<BenchmarkCase::Input> inputs;
-    auto st = state::name;
-    std::string input_name;
-    bytes input;
-    for (std::string l; std::getline(inputs_file, l);)
+    inputs.reserve(state_test.multi_tx.inputs.size());
+    for (size_t i = 0; i < state_test.multi_tx.inputs.size(); ++i)
     {
-        switch (st)
-        {
-        case state::name:
-            if (l.empty())
-                continue;  // Skip any empty line.
-            input_name = std::move(l);
-            st = state::input;
-            break;
-
-        case state::input:
-            input = from_hexx(l);
-            st = state::expected_output;
-            break;
-
-        case state::expected_output:
-            inputs.emplace_back(std::move(input_name), std::move(input), from_hexx(l));
-            input_name = {};
-            input = {};
-            st = state::name;
-            break;
-        }
+        inputs.emplace_back(
+            BenchmarkCase::Input{state_test.input_labels.at(i), state_test.multi_tx.inputs[i]});
     }
-
     return inputs;
 }
 
 /// Loads a benchmark case from a file at `path` and all its inputs from the matching inputs file.
 BenchmarkCase load_benchmark(const fs::path& path, const std::string& name_prefix)
 {
-    const auto name = name_prefix + path.stem().string();
+    auto state_test = evmone::test::load_state_test(path);
 
-    std::ifstream file{path};
-    std::string code_hexx{std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{}};
-    BenchmarkCase b{name, from_hexx(code_hexx)};
-
-    auto inputs_path = path;
-    inputs_path.replace_extension(inputs_extension);
-    if (fs::exists(inputs_path))
-        b.inputs = load_inputs(inputs_path);
+    const auto name = name_prefix + path.stem().string();
+    const auto code = state_test.pre_state.get(state_test.multi_tx.to.value()).code;
+    const auto inputs = load_inputs(state_test);
 
-    return b;
+    return BenchmarkCase{name, code, inputs};
 }
 
 /// Loads all benchmark cases from the given directory and all its subdirectories.
@@ -124,7 +80,7 @@ std::vector<BenchmarkCase> load_benchmarks_from_dir(  // NOLINT(misc-no-recursio
     {
         if (e.is_directory())
             subdirs.emplace_back(e);
-        else if (e.path().extension() == runtime_code_extension)
+        else if (e.path().extension() == ".json")
             code_files.emplace_back(e);
     }
 
@@ -291,13 +247,12 @@ std::tuple<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)
     if (!code_hex_file.empty())
     {
         std::ifstream file{code_hex_file};
-        BenchmarkCase b{code_hex_file,
-            from_spaced_hex(std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{})
-                .value()};
-        b.inputs.emplace_back(
-            "", from_hex(input_hex).value(), from_hex(expected_output_hex).value());
-
-        return {0, {std::move(b)}};
+        return {0, {BenchmarkCase{code_hex_file,
+                       from_spaced_hex(
+                           std::istreambuf_iterator<char>{file}, std::istreambuf_iterator<char>{})
+                           .value(),
+                       {BenchmarkCase::Input{"", from_hex(input_hex).value(),
+                           from_hex(expected_output_hex).value()}}}}};
     }
 
     return {0, {}};