From 18858bb4960a352d44291f5136d658d8ade81185 Mon Sep 17 00:00:00 2001 From: Dominik Berner Date: Thu, 26 Dec 2019 23:48:14 +0100 Subject: [PATCH] Add support for catch2 benchmarks --- .github/workflows/ci.yml | 35 +++++++++++++++ src/config.ts | 4 +- src/extract.ts | 66 +++++++++++++++++++++++++++++ src/write.ts | 2 + test/data/extract/catch2_output.txt | 28 ++++++++++++ test/extract.ts | 19 +++++++++ 6 files changed, 152 insertions(+), 2 deletions(-) create mode 100644 test/data/extract/catch2_output.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c01221f28..fbc055e30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,41 @@ jobs: skip-fetch-gh-pages: true fail-on-alert: true - run: node ./scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + - uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npm run build + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: "catch2" + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + - run: node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + only-alert-with-cache: name: Run alert check with actions/cache runs-on: ubuntu-latest diff --git a/src/config.ts b/src/config.ts index a07e30285..64de1ff93 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import * as os from 'os'; import * as path from 'path'; -export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp'; +export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2'; export interface Config { name: string; tool: ToolType; @@ -22,7 +22,7 @@ export interface Config { maxItemsInChart: number | null; } -export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp']; +export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2']; const RE_UINT = /^\d+$/; function validateToolType(tool: string): asserts tool is ToolType { diff --git a/src/extract.ts b/src/extract.ts index 5fb077e07..856278932 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -304,6 +304,69 @@ function extractGoogleCppResult(output: string): BenchmarkResult[] { }); } +function extractCatch2Result(output: string): BenchmarkResult[] { + const lines = output.split('\n'); + + const ret = []; + // Example: + // test bench_fib_20 ... bench: 37,174 ns/iter (+/- 7,527) + const reTestCaseStart = /^(benchmark name) +(samples) +(iterations) +(estimated)/; + const reBenchmarkStart = /^([a-zA-Z\d ]+) +([\d]+) +([\d]+) +([\d]+\.[\d]+) (ms|us)/; + const reBenchmarkValues = /^ +([\d]+\.[\d]+) (us|ms) +([\d]+\.[\d]+) (us|ms) +([\d]+\.[\d]+) (us|ms)/; + + let benchmarkNr = -1; + let testCaseNr = -1; + + const currentValue = 0; + const currentUnit = ''; + const currentRange = ''; + let linesSinceBenchmarkStart = -1; + + for (const line of lines) { + const m = line.match(reTestCaseStart); + if (m != null) { + testCaseNr++; + } + // no benchmark section found so far, ignore + if (testCaseNr < 0) { + continue; + } + + if (benchmarkNr >= 0) { + linesSinceBenchmarkStart++; + } + + const benchmarkValueMatch = line.match(reBenchmarkValues); + if (benchmarkValueMatch === null && linesSinceBenchmarkStart == 1) { + throw new Error( + 'Retrieved a catch2 benchmark but no values for it\nCatch2 result file is possibly mangled\n\n' + line, + ); + } + if (linesSinceBenchmarkStart == 1 && benchmarkValueMatch != null) { + ret[benchmarkNr].value = parseFloat(benchmarkValueMatch[1]); + ret[benchmarkNr].unit = benchmarkValueMatch[2]; + } + if (linesSinceBenchmarkStart == 2 && benchmarkValueMatch != null) { + ret[benchmarkNr].range = '+/- ' + benchmarkValueMatch[1].trim(); + } + + const benchmarkMatch = line.match(reBenchmarkStart); + if (benchmarkMatch != null) { + linesSinceBenchmarkStart = 0; + benchmarkNr++; + ret.push({ + name: benchmarkMatch[1].trim(), + value: currentValue, + range: currentRange, + unit: currentUnit, + extra: benchmarkMatch[2] + ' samples', + }); + } + } + + return ret; +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool } = config; @@ -325,6 +388,9 @@ export async function extractResult(config: Config): Promise { case 'googlecpp': benches = extractGoogleCppResult(output); break; + case 'catch2': + benches = extractCatch2Result(output); + break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); } diff --git a/src/write.ts b/src/write.ts index 647277c1e..c76cb1f0c 100644 --- a/src/write.ts +++ b/src/write.ts @@ -69,6 +69,8 @@ function biggerIsBetter(tool: ToolType): boolean { return true; case 'googlecpp': return false; + case 'catch2': + return false; } } diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output.txt new file mode 100644 index 000000000..e54f30e53 --- /dev/null +++ b/test/data/extract/catch2_output.txt @@ -0,0 +1,28 @@ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catch2_bench is a Catch v2.11.0 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Fibonacci +------------------------------------------------------------------------------- +/home/doeme/Code/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci 20 100 2 8.4318 ms + 43.186 us 41.402 us 46.246 us + 11.719 us 7.847 us 17.747 us + +Fibonacci 25 100 1 45.6213 ms + 451.183 us 441.654 us 469.296 us + 65.064 us 36.524 us 106.192 us + + +=============================================================================== +test cases: 1 | 1 passed +assertions: - none - + diff --git a/test/extract.ts b/test/extract.ts index ae5ff5d35..876735736 100644 --- a/test/extract.ts +++ b/test/extract.ts @@ -54,6 +54,25 @@ describe('extractResult()', function() { }, ], }, + { + tool: 'catch2', + expected: [ + { + name: 'Fibonacci 20', + range: '+/- 11.719', + unit: 'us', + value: 43.186, + extra: '100 samples', + }, + { + name: 'Fibonacci 25', + range: '+/- 65.064', + unit: 'us', + value: 451.183, + extra: '100 samples', + }, + ], + }, { tool: 'go', expected: [