diff --git a/.github/workflows/catch2.yml b/.github/workflows/catch2.yml new file mode 100644 index 000000000..4b0b6f33f --- /dev/null +++ b/.github/workflows/catch2.yml @@ -0,0 +1,29 @@ +name: Catch2 C++ Example +on: push + +jobs: + benchmark: + name: Run C++ benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: "catch2" + output-file-path: examples/catch2/benchmark_result.txt + # Use personal access token instead of GITHUB_TOKEN due to https://github.community/t5/GitHub-Actions/Github-action-not-triggering-gh-pages-upon-push/td-p/26869/highlight/false + github-token: ${{ secrets.PERSONAL_GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: "200%" + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: "@bernedom" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c01221f28..fbc055e30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,41 @@ jobs: skip-fetch-gh-pages: true fail-on-alert: true - run: node ./scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + - uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npm run build + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: "catch2" + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + - run: node ./scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + only-alert-with-cache: name: Run alert check with actions/cache runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3aaea1cc9..88272d65b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,7 +3,7 @@ Development ## How to add new benchmark tool support -Adding support for new benchmaking tools is welcome! +Adding support for new benchmarking tools is welcome! 1. Add your tool name in `src/config.ts` 2. Implement the logic to extract benchmark results from output in `src/extract.ts` @@ -11,7 +11,7 @@ Adding support for new benchmaking tools is welcome! 4. Add your tool's color in `default_index_html.ts` 5. Add example project under `examples/` directory 6. Add workflow to run the example project under `.github/workflows/` directory -7. Update `.github/workflows/ci.yml` to check your tool works without an error +7. Update `.github/workflows/ci.yml` to check your tool works without an error (see below for needed changes) 8. Add README.md in the example project directory and update README.md at root directory Important part is 2. @@ -29,6 +29,22 @@ And for another example, here are commits to add support for `pytest-benchmark`: - Add workflows for test and example: https://github.com/rhysd/github-action-benchmark/commit/1e4ebf2e9ecde9e7620661c60455b22837a2bdaf - Add documentation: https://github.com/rhysd/github-action-benchmark/commit/895f92f564521597492bd281cbf6c8efd39f628e +## Running CI workflow on a forked repo + +Since the benchmark data includes the URL of the repository the tests and examples will fail when the repo is forked. In order to get the ci workflow and all examples running the URL has to be changed + +### Change workflows and tests + +1. in [`ci_validate_modification.ts`](scripts/ci_validate_modification.ts) in the `validateJSON` function replace `https://github.com/rhysd/github-action-benchmark` with your repo URL (i.e. `https://github.com//github-action-benchmark`) +2. in all workflow (`.yml`) in `.github/workflows` replace `alert-comment-cc-users: '@rhysd'` with your user + +### Adapt past benchmark data to new repo path + +1. checkout the branch `gh-pages` +2. in `dev/bench/data.js` replace `https://github.com/rhysd/github-action-benchmark` with your repo URL (i.e. `https://github.com//github-action-benchmark`) + +In case you are adding a new tool and wand to run ci, you have to add at least one set of example data to the `data.js` in the gh-pages branch. + ## How to create a new release 1. Run `$ bash scripts/prepare-release.sh v1` diff --git a/examples/catch2/.gitignore b/examples/catch2/.gitignore new file mode 100644 index 000000000..d16386367 --- /dev/null +++ b/examples/catch2/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/examples/catch2/CMakeLists.txt b/examples/catch2/CMakeLists.txt new file mode 100644 index 000000000..6ed2214c2 --- /dev/null +++ b/examples/catch2/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.12) +project("Catch2_bench") + +include(FetchContent) + +FetchContent_Declare( + catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v2.11.0) + +FetchContent_GetProperties(catch2) +if(NOT catch2_POPULATED) + FetchContent_Populate(catch2) + add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR}) +endif() + +add_executable(${PROJECT_NAME}) +target_sources(${PROJECT_NAME} PRIVATE catch2_bench.cpp) +target_link_libraries(${PROJECT_NAME} Catch2::Catch2) + +target_compile_options( + ${PROJECT_NAME} + PRIVATE + $<$:/DCATCH_CONFIG_ENABLE_BENCHMARKING> + $<$,$,$>:-DCATCH_CONFIG_ENABLE_BENCHMARKING> +) diff --git a/examples/catch2/catch2_bench.cpp b/examples/catch2/catch2_bench.cpp new file mode 100644 index 000000000..cab08bf83 --- /dev/null +++ b/examples/catch2/catch2_bench.cpp @@ -0,0 +1,11 @@ +#include "fib.hpp" +#define CATCH_CONFIG_MAIN +#include + +TEST_CASE("Fibonacci") { + + // now let's benchmark: + BENCHMARK("Fibonacci 20") { return fib(20); }; + + BENCHMARK("Fibonacci 25") { return fib(25); }; +} \ No newline at end of file diff --git a/examples/catch2/fib.hpp b/examples/catch2/fib.hpp new file mode 100644 index 000000000..559e886c8 --- /dev/null +++ b/examples/catch2/fib.hpp @@ -0,0 +1,11 @@ +#if !defined FIB_HPP_INCLUDED +#define FIB_HPP_INCLUDED + +int fib(int const i) { + if (i <= 1) { + return 1; + } + return fib(i - 2) + fib(i - 1); +} + +#endif // FIB_HPP_INCLUDED diff --git a/src/config.ts b/src/config.ts index a07e30285..64de1ff93 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,7 @@ import { promises as fs } from 'fs'; import * as os from 'os'; import * as path from 'path'; -export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp'; +export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest' | 'googlecpp' | 'catch2'; export interface Config { name: string; tool: ToolType; @@ -22,7 +22,7 @@ export interface Config { maxItemsInChart: number | null; } -export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp']; +export const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest', 'googlecpp', 'catch2']; const RE_UINT = /^\d+$/; function validateToolType(tool: string): asserts tool is ToolType { diff --git a/src/default_index_html.ts b/src/default_index_html.ts index 94e98ac6e..e65c50223 100644 --- a/src/default_index_html.ts +++ b/src/default_index_html.ts @@ -116,6 +116,7 @@ export const DEFAULT_INDEX_HTML = String.raw` benchmarkjs: '#f1e05a', pytest: '#3572a5', googlecpp: '#f34b7d', + catch2: '#f34b7d', _: '#333333' }; diff --git a/src/extract.ts b/src/extract.ts index 5fb077e07..018ddeac8 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -304,6 +304,82 @@ function extractGoogleCppResult(output: string): BenchmarkResult[] { }); } +function extractCatch2Result(output: string): BenchmarkResult[] { + const lines = output.split('\n'); + + const ret = []; + // Example: + + // benchmark name samples iterations estimated <-- Start benchmark section + // mean low mean high mean <-- Ignored + // std dev low std dev high std dev <-- Ignored + // ----------------------------------------------------- <-- Ignored + // Fibonacci 20 100 2 8.4318 ms <-- Start actual benchmark + // 43.186 us 41.402 us 46.246 us <-- Actual benchmark data + // 11.719 us 7.847 us 17.747 us <-- Ignored + + const reTestCaseStart = /^benchmark name +samples +iterations +estimated/; + const reBenchmarkStart = /^([a-zA-Z\d ]+) +(\d+) +(\d+) +(\d+(\.\d+)?) (ns|ms|us|s)/; + const reBenchmarkValues = /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s) +(\d+(?:\.\d+)?) (ns|us|ms|s)/; + + let benchmarkNr = -1; + let testCaseNr = -1; + + let linesSinceBenchmarkStart = -1; + + for (const line of lines) { + const m = line.match(reTestCaseStart); + if (m !== null) { + testCaseNr++; + } + // no benchmark section found so far, ignore + if (testCaseNr < 0) { + continue; + } + + if (benchmarkNr >= 0) { + linesSinceBenchmarkStart++; + } + + const benchmarkValueMatch = line.match(reBenchmarkValues); + if (benchmarkValueMatch === null && linesSinceBenchmarkStart === 1) { + throw new Error( + 'Retrieved a catch2 benchmark but no values for it\nCatch2 result file is possibly mangled\n\n' + line, + ); + } + if (linesSinceBenchmarkStart === 1 && benchmarkValueMatch !== null) { + ret[benchmarkNr].value = parseFloat(benchmarkValueMatch[1]); + ret[benchmarkNr].unit = benchmarkValueMatch[2]; + } + if (linesSinceBenchmarkStart === 2 && benchmarkValueMatch !== null) { + ret[benchmarkNr].range = '+/- ' + benchmarkValueMatch[1].trim(); + } + + const benchmarkMatch = line.match(reBenchmarkStart); + if (benchmarkMatch !== null) { + linesSinceBenchmarkStart = 0; + benchmarkNr++; + ret.push({ + name: benchmarkMatch[1].trim(), + value: 0, + range: '', + unit: '', + extra: benchmarkMatch[2] + ' samples', + }); + } + } + + if ( + ret.every(function(r) { + return r.range === '' && r.unit === ''; + }) + ) { + throw new Error(`Invalid range or unit for catch2 benchmark`); + } + + return ret; +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool } = config; @@ -325,6 +401,9 @@ export async function extractResult(config: Config): Promise { case 'googlecpp': benches = extractGoogleCppResult(output); break; + case 'catch2': + benches = extractCatch2Result(output); + break; default: throw new Error(`FATAL: Unexpected tool: '${tool}'`); } diff --git a/src/write.ts b/src/write.ts index 647277c1e..c76cb1f0c 100644 --- a/src/write.ts +++ b/src/write.ts @@ -69,6 +69,8 @@ function biggerIsBetter(tool: ToolType): boolean { return true; case 'googlecpp': return false; + case 'catch2': + return false; } } diff --git a/test/data/extract/catch2_output.txt b/test/data/extract/catch2_output.txt new file mode 100644 index 000000000..29037ef30 --- /dev/null +++ b/test/data/extract/catch2_output.txt @@ -0,0 +1,32 @@ + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Catch2_bench is a Catch v2.11.0 host application. +Run with -? for options + +------------------------------------------------------------------------------- +Fibonacci +------------------------------------------------------------------------------- +/home/doeme/Code/github-action-benchmark/examples/catch2/catch2_bench.cpp:5 +............................................................................... + +benchmark name samples iterations estimated + mean low mean high mean + std dev low std dev high std dev +------------------------------------------------------------------------------- +Fibonacci 20 100 2 8.4318 ms + 43.186 us 41.402 us 46.246 us + 11.719 us 7.847 us 17.747 us + +Fibonacci 25 100 1 45.6213 ms + 451.183 us 441.654 us 469.296 us + 65.064 us 36.524 us 106.192 us + +Fibonacci Integer 100 1 45 ms + 123 s 441 s 296.123 ns + 2 s 36.524 us 106.192 us + + +=============================================================================== +test cases: 1 | 1 passed +assertions: - none - + diff --git a/test/extract.ts b/test/extract.ts index ae5ff5d35..d896dcb2d 100644 --- a/test/extract.ts +++ b/test/extract.ts @@ -54,6 +54,32 @@ describe('extractResult()', function() { }, ], }, + { + tool: 'catch2', + expected: [ + { + name: 'Fibonacci 20', + range: '+/- 11.719', + unit: 'us', + value: 43.186, + extra: '100 samples', + }, + { + name: 'Fibonacci 25', + range: '+/- 65.064', + unit: 'us', + value: 451.183, + extra: '100 samples', + }, + { + name: 'Fibonacci Integer', + range: '+/- 2', + unit: 's', + value: 123, + extra: '100 samples', + }, + ], + }, { tool: 'go', expected: [