diff --git a/scripts/benchmark-runner.py b/scripts/benchmark-runner.py new file mode 100755 index 000000000000..31b446cb2d65 --- /dev/null +++ b/scripts/benchmark-runner.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import os +import sys +import tempfile +from veloxbench.veloxbench.cpp_micro_benchmarks import LocalCppMicroBenchmarks + +_OUTPUT_NUM_COLS = 100 + + +# Cosmetic helper functions. +def color_red(text) -> str: + return "\033[91m{}\033[00m".format(text) if sys.stdout.isatty() else text + + +def color_yellow(text) -> str: + return "\033[93m{}\033[00m".format(text) if sys.stdout.isatty() else text + + +def color_green(text) -> str: + return "\033[92m{}\033[00m".format(text) if sys.stdout.isatty() else text + + +def bold(text) -> str: + return "\033[1m{}\033[00m".format(text) if sys.stdout.isatty() else text + + +def get_benchmark_handle(file_path, name): + if name[0] == "%": + name = name[1:] + return "{}/{}".format(os.path.basename(file_path), name) + + +def fmt_runtime(time_ns): + if time_ns < 1000: + return "{:.2f}ns".format(time_ns) + + time_usec = time_ns / 1000 + if time_usec < 1000: + return "{:.2f}us".format(time_usec) + else: + return "{:.2f}ms".format(time_usec / 1000) + + +def compare_file(args, target_data, baseline_data): + baseline_map = {} + for row in baseline_data: + baseline_map[get_benchmark_handle(row[0], row[1])] = row[2] + + passes = [] + faster = [] + failures = [] + + for row in target_data: + # Folly benchmark exports line separators by mistake as an entry on + # the json file. + if row[1] == "-": + continue + + benchmark_handle = get_benchmark_handle(row[0], row[1]) + baseline_result = baseline_map[benchmark_handle] + target_result = row[2] + + if baseline_result == 0 or target_result == 0: + delta = 0 + elif baseline_result > target_result: + delta = 1 - (target_result / baseline_result) + else: + delta = (1 - (baseline_result / target_result)) * -1 + + if abs(delta) > args.threshold: + if delta > 0: + status = color_green("🗲 Pass") + passes.append(benchmark_handle) + faster.append(benchmark_handle) + else: + status = color_red("✗ Fail") + failures.append(benchmark_handle) + else: + status = color_green("✓ Pass") + passes.append(benchmark_handle) + + suffix = "({} vs {}) {:+.2f}%".format( + fmt_runtime(baseline_result), fmt_runtime(target_result), delta * 100 + ) + + # Prefix length is 12 bytes (considering utf8 and invisible chars). + spacing = " " * (_OUTPUT_NUM_COLS - (12 + len(benchmark_handle) + len(suffix))) + print(" {}: {}{}{}".format(status, benchmark_handle, spacing, suffix)) + + return passes, faster, failures + + +def find_json_files(path): + json_files = {} + with os.scandir(path) as files: + for file_found in files: + if file_found.name.endswith(".json"): + json_files[file_found.name] = file_found.path + return json_files + + +def compare(args): + print( + "=> Starting comparison using {} ({}%) as threshold.".format( + args.threshold, args.threshold * 100 + ) + ) + print("=> Values are reported as percentage normalized to the largest values:") + print("=> (positive means speedup; negative means regression).") + + # Read file lists from both directories. + baseline_map = find_json_files(args.baseline_path) + target_map = find_json_files(args.target_path) + + all_passes = [] + all_faster = [] + all_failures = [] + + # Compare json results from each file. + for file_name, target_path in target_map.items(): + print("=" * _OUTPUT_NUM_COLS) + print(file_name) + print("=" * _OUTPUT_NUM_COLS) + + if file_name not in baseline_map: + print("WARNING: baseline file for '%s' not found. Skipping." % file_name) + continue + + # Open and read each file. + with open(target_path) as f: + target_data = json.load(f) + + with open(baseline_map[file_name]) as f: + baseline_data = json.load(f) + + passes, faster, failures = compare_file(args, target_data, baseline_data) + all_passes += passes + all_faster += faster + all_failures += failures + + def print_list(names): + for n in names: + print(" %s" % n) + + # Print a nice summary of the results: + print("Summary:") + if all_passes: + faster_summary = ( + " ({} are faster):".format(len(all_faster)) if all_faster else "" + ) + print(color_green(" Pass: %d%s " % (len(all_passes), faster_summary))) + print_list(all_faster) + + if all_failures: + print(color_red(" Fail: %d" % len(all_failures))) + print_list(all_failures) + return 1 + return 0 + + +def run(args): + LocalCppMicroBenchmarks().run( + output_dir=args.output_path or tempfile.mkdtemp(), + binary_path=args.binary_path, + binary_filter=args.binary_filter, + bm_filter=args.bm_filter, + bm_max_secs=args.bm_max_secs, + bm_max_trials=args.bm_max_trials, + bm_estimate_time=args.bm_estimate_time, + ) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Velox Benchmark Runner Utility.") + parser.set_defaults(func=lambda _: parser.print_help()) + + subparsers = parser.add_subparsers(help="Please specify one of the subcommands.") + + # Arguments for the "run" subparser. + parser_run = subparsers.add_parser("run", help="Run benchmarks and dump results.") + parser_run.add_argument( + "--binary_path", + default=None, + help="Directory where benchmark binaries are stored. " + "Defaults to release build directory.", + ) + parser_run.add_argument( + "--output_path", + default=None, + help="Directory where output json files will be written to. " + "By default generate a temporary directory.", + ) + parser_run.add_argument( + "--binary_filter", + default=None, + help="Filter applied to binary names. " + "By default execute all binaries found.", + ) + parser_run.add_argument( + "--bm_filter", + default=None, + help="Filter applied to benchmark names within binaries. " + "By default execute all benchmarks.", + ) + parser_run.add_argument( + "--bm_max_secs", + default=None, + type=int, + help="For how many seconds to run each benchmark in a binary.", + ) + parser_run.add_argument( + "--bm_max_trials", + default=None, + type=int, + help="Maximum number of trials (iterations) executed for each benchmark.", + ) + parser_run.add_argument( + "--bm_estimate_time", + default=False, + action="store_true", + help="Use folly benchmark --bm_estimate_time flag.", + ) + parser_run.set_defaults(func=run) + + # Arguments for the "compare" subparser. + parser_compare = subparsers.add_parser( + "compare", help="Compare benchmark dumped results." + ) + parser_compare.set_defaults(func=compare) + parser_compare.add_argument( + "--baseline_path", + required=True, + help="Path where containing base dump results.", + ) + parser_compare.add_argument( + "--target_path", + required=True, + help="Path where containing target dump results.", + ) + parser_compare.add_argument( + "-t", + "--threshold", + type=float, + default=0.05, + help="Comparison threshold. " + "Variations larger than this threshold will be reported as failures. " + "Default 0.05 (5%%).", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/veloxbench/__init__.py b/scripts/veloxbench/__init__.py new file mode 100644 index 000000000000..8daf2005df70 --- /dev/null +++ b/scripts/veloxbench/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/scripts/veloxbench/veloxbench/cpp_micro_benchmarks.py b/scripts/veloxbench/veloxbench/cpp_micro_benchmarks.py index aaf1f3f203db..d27796d79207 100755 --- a/scripts/veloxbench/veloxbench/cpp_micro_benchmarks.py +++ b/scripts/veloxbench/veloxbench/cpp_micro_benchmarks.py @@ -63,7 +63,7 @@ class LocalCppMicroBenchmarks: def run( self, - result_dir, + output_dir, binary_path=None, binary_filter=None, bm_filter=None, @@ -78,13 +78,14 @@ def run( binary_path = self._default_binary_path() binaries = self._find_binaries(binary_path) - result_dir_path = pathlib.Path(result_dir) + output_dir_path = pathlib.Path(output_dir) + output_dir_path.mkdir(parents=True, exist_ok=True) for binary_path in binaries: if binary_filter and not re.search(binary_filter, binary_path.name): continue - out_path = result_dir_path / f"{binary_path.name}.json" + out_path = output_dir_path / f"{binary_path.name}.json" print(f"Executing and dumping results for '{binary_path}' to '{out_path}':") run_command = [ binary_path, @@ -167,67 +168,3 @@ def _format_unit(x): if x == "items_per_second": return "i/s" return x - - -def parse_arguments(): - parser = argparse.ArgumentParser( - description="VeloxBench Client Tool", - epilog="(c) Meta Platforms 2004-present", - ) - parser.add_argument( - "--binary_path", - default=None, - help="Directory where benchmark binaries are stored. " - "Defaults to release build directory.", - ) - parser.add_argument( - "--binary_filter", - default=None, - help="Filter applied to binary names. " - "By default execute all binaries found.", - ) - parser.add_argument( - "--bm_filter", - default=None, - help="Filter applied to benchmark names within binaries. " - "By default execute all benchmarks.", - ) - parser.add_argument( - "--bm_max_secs", - default=None, - type=int, - help="For how many second to run each benchmark in a binary.", - ) - parser.add_argument( - "--bm_max_trials", - default=None, - type=int, - help="Maximum number of trials (iterations) executed for each benchmark.", - ) - parser.add_argument( - "--bm_estimate_time", - default=False, - action="store_true", - help="Use folly benchmark --bm_estimate_time flag.", - ) - return parser.parse_args() - - -def main(): - args = parse_arguments() - - with tempfile.TemporaryDirectory() as result_dir: - LocalCppMicroBenchmarks().run( - result_dir=result_dir, - binary_path=args.binary_path, - binary_filter=args.binary_filter, - bm_filter=args.bm_filter, - bm_max_secs=args.bm_max_secs, - bm_max_trials=args.bm_max_trials, - bm_estimate_time=args.bm_estimate_time, - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/veloxbench/veloxbench/implemented_benchmarks.py b/scripts/veloxbench/veloxbench/implemented_benchmarks.py index 3dc2941d695e..5220f3bdcb1d 100644 --- a/scripts/veloxbench/veloxbench/implemented_benchmarks.py +++ b/scripts/veloxbench/veloxbench/implemented_benchmarks.py @@ -25,14 +25,14 @@ @conbench.runner.register_benchmark class RecordCppMicroBenchmarks(LocalCppMicroBenchmarks, Benchmark): def run(self, **kwargs): - with tempfile.TemporaryDirectory() as result_dir: + with tempfile.TemporaryDirectory() as output_dir: # run benchmarks and save to a tempdir super().run( - result_dir=result_dir, bm_max_secs=10, bm_max_trials=1000000, **kwargs + output_dir=output_dir, bm_max_secs=10, bm_max_trials=1000000, **kwargs ) # iterate through files to make the suites - with os.scandir(result_dir) as result_files: + with os.scandir(output_dir) as result_files: for result_file in result_files: suite = result_file.name.replace(".json", "", 1) with open(result_file.path, "r") as f: