From 306dcb4c0cf467e7f38efde8334e060c4c2581d6 Mon Sep 17 00:00:00 2001 From: Marvin Borner Date: Tue, 25 Feb 2025 22:27:09 +0100 Subject: [PATCH] Better debugging by grouping logs --- generate/benchmarks.sh | 8 +++++--- generate/main.sh | 4 +++- generate/reference/are-we-fast-yet.sh | 19 +++++++++++++------ generate/reference/duality-of-compilation.sh | 19 +++++++++++++------ generate/reference/effect-handlers-bench.sh | 13 +++++++++---- generate/reference/effekt.sh | 8 +++++--- 6 files changed, 48 insertions(+), 23 deletions(-) diff --git a/generate/benchmarks.sh b/generate/benchmarks.sh index 1f2f858..3971396 100755 --- a/generate/benchmarks.sh +++ b/generate/benchmarks.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" # we need to cd here since some examples import local files cd ../effekt/ @@ -16,7 +16,7 @@ done # then build and run all with JSON phase timings for file in $FILES; do - effekt.sh -o out/ --time json "$file" &>/dev/null + effekt.sh -o out/ --time json "$file" >&2 done # now also measure execution time of backends based on benchmark configuration @@ -39,7 +39,7 @@ benchmark() { effekt.sh --backend "$backend" -b "$file" for prerun in $(seq $PRERUNS); do - ./"$outfile" "${arr[1]}" &>/dev/null + ./"$outfile" "${arr[1]}" >&2 done total_time=0 @@ -64,3 +64,5 @@ benchmark() { for backend in $BACKENDS; do benchmark "$backend" done + +>&2 echo "::endgroup::" diff --git a/generate/main.sh b/generate/main.sh index 0156cdb..3817909 100755 --- a/generate/main.sh +++ b/generate/main.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" ./build.sh | ./append.sh build ./reference.sh | ./append.sh reference @@ -19,3 +19,5 @@ find * -type f -name "*.json" -printf "\"%p\"\n" | jq -s . >index.json # reset effekt repository git -C ../effekt/ checkout . + +>&2 echo "::endgroup::" diff --git a/generate/reference/are-we-fast-yet.sh b/generate/reference/are-we-fast-yet.sh index da7553e..ef62f47 100755 --- a/generate/reference/are-we-fast-yet.sh +++ b/generate/reference/are-we-fast-yet.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" # are-we-fast-yet uses different filenames than we do! TRACKED="Bounce List Mandelbrot NBody Permute Queens Sieve Storage Towers" @@ -19,7 +19,7 @@ tmpfile=$(mktemp /tmp/hyperfine_are-we-fast-yet.XXXXX) # command -> json benchmark() { - if hyperfine --export-json "$tmpfile" "$1" &>/dev/null; then + if hyperfine --export-json "$tmpfile" "$1" >&2; then jq "{mean: .results[0].mean, stddev: .results[0].stddev, arg: 0}" "$tmpfile" else echo "{\"arg\": 0}" @@ -30,7 +30,7 @@ benchmark() { echo "{" # --- JavaScript benchmarks --- ->&2 echo "JavaScript" +>&2 echo "::group::js" cd reference/are-we-fast-yet/benchmarks/JavaScript/ echo "\"js\":" @@ -45,9 +45,10 @@ i=0 } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- Python benchmarks --- ->&2 echo "Python" +>&2 echo "::group::python" cd ../Python echo "\"python\":" @@ -62,9 +63,10 @@ i=0 } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- Java benchmarks --- ->&2 echo "Java" +>&2 echo "::group::java" cd ../Java ./build.sh >&2 @@ -81,9 +83,10 @@ i=0 } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- C++ benchmarks --- ->&2 echo "C++" +>&2 echo "::group::cpp" cd ../C++ # generated using the build.sh script -- TODO: Do we want to change this? @@ -100,8 +103,12 @@ i=0 done } | jq -s 'add' +>&2 echo "::endgroup::" + # --- Done --- # output end echo "}" rm "$tmpfile" + +>&2 echo "::endgroup::" diff --git a/generate/reference/duality-of-compilation.sh b/generate/reference/duality-of-compilation.sh index 0515713..20f0a75 100755 --- a/generate/reference/duality-of-compilation.sh +++ b/generate/reference/duality-of-compilation.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" TRACKED="erase_unused factorial_accumulator fibonacci_recursive iterate_increment lookup_tree match_options sum_range" PREFIX="duality_of_compilation" @@ -18,7 +18,7 @@ get_arg() { # command -> arg -> json benchmark() { - if hyperfine --export-json "$tmpfile" "$1" &>/dev/null; then + if hyperfine --export-json "$tmpfile" "$1" >&2; then jq "{mean: .results[0].mean, stddev: .results[0].stddev, arg: $2}" "$tmpfile" else echo "{\"arg\": $2}" @@ -29,7 +29,7 @@ benchmark() { echo "{" # --- Rust benchmarks --- ->&2 echo "Rust" +>&2 echo "::group::rust" cd reference/duality-of-compilation/rust/ make >&2 @@ -45,9 +45,10 @@ echo "\"rust\":" } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- Koka benchmarks --- ->&2 echo "Koka" +>&2 echo "::group::koka" cd ../koka/ # TODO: make this smarter (for CI) @@ -64,9 +65,10 @@ echo "\"koka\":" } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- mlton benchmarks --- ->&2 echo "mlton" +>&2 echo "::group::mlton" cd ../mlton/ make >&2 @@ -82,9 +84,10 @@ echo "\"mlton\":" } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- Ocaml benchmarks --- ->&2 echo "ocaml" +>&2 echo "::group::ocaml" cd ../ocaml/ make >&2 @@ -99,8 +102,12 @@ echo "\"ocaml\":" done } | jq -s 'add' +>&2 echo "::endgroup::" + # --- Done --- # output end echo "}" rm "$tmpfile" + +>&2 echo "::endgroup::" diff --git a/generate/reference/effect-handlers-bench.sh b/generate/reference/effect-handlers-bench.sh index 14d2f5f..cf59f9f 100755 --- a/generate/reference/effect-handlers-bench.sh +++ b/generate/reference/effect-handlers-bench.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" TRACKED="countdown fibonacci_recursive generator iterator nqueens parsing_dollars product_early resume_nontail tree_explore triples" PREFIX="effect_handlers_bench" @@ -18,7 +18,7 @@ get_arg() { # command -> arg -> json benchmark() { - if hyperfine --export-json "$tmpfile" "$1" &>/dev/null; then + if hyperfine --export-json "$tmpfile" "$1" >&2; then jq "{mean: .results[0].mean, stddev: .results[0].stddev, arg: $2}" "$tmpfile" else echo "{\"arg\": $2}" @@ -29,7 +29,7 @@ benchmark() { echo "{" # --- Ocaml benchmarks --- ->&2 echo "Ocaml" +>&2 echo "::group::ocaml" cd reference/effect-handlers-bench/benchmarks/ocaml/ eval $(opam env --switch=4.12.0+domains+effects --set-switch) @@ -47,9 +47,10 @@ echo "\"ocaml\":" } | jq -s 'add' echo "," +>&2 echo "::endgroup::" # --- Eff benchmarks --- ->&2 echo "Eff" +>&2 echo "::group::eff" cd ../eff/ eval $(opam env --switch=/tmp/runner/effekt-plots/effekt-plots --set-switch) @@ -66,6 +67,8 @@ echo "\"eff\":" done } | jq -s 'add' +>&2 echo "::endgroup::" + # --- Koka benchmarks --- # TODO: This uses a very old version and is broken with current @@ -75,3 +78,5 @@ echo "\"eff\":" # output end echo "}" rm "$tmpfile" + +>&2 echo "::endgroup::" diff --git a/generate/reference/effekt.sh b/generate/reference/effekt.sh index 778712f..ccd0bb1 100755 --- a/generate/reference/effekt.sh +++ b/generate/reference/effekt.sh @@ -1,7 +1,7 @@ #!/bin/env bash set -e ->&2 echo "$0" +>&2 echo "::group::$0" cd ../effekt/ @@ -11,7 +11,7 @@ BACKENDS="llvm js" tmpfile=$(mktemp /tmp/hyperfine_effekt.XXXXX) benchmark() { - if hyperfine --export-json "$tmpfile" "$1" &>/dev/null; then + if hyperfine --export-json "$tmpfile" "$1" >&2; then jq "{mean: .results[0].mean, stddev: .results[0].stddev, arg: $2}" "$tmpfile" else echo "{\"arg\": $2}" @@ -29,7 +29,7 @@ benchmark() { outfile="out/$(basename "$file" .effekt)" echo "{\"$filename\":" - effekt.sh --backend "$backend" -b "$file" &>/dev/null + effekt.sh --backend "$backend" -b "$file" >&2 arg=${arr[1]} benchmark "./$outfile $arg" "$arg" echo "}" @@ -40,3 +40,5 @@ benchmark() { } | jq -s 'add' rm "$tmpfile" + +>&2 echo "::endgroup::"