support pytest-benchmark

action.yml

@@ -11,7 +11,7 @@ inputs:
     required: true
     default: 'Benchmark'
   tool:
-    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs"'
+    description: 'Tool to use get benchmark output. One of "cargo", "go", "benchmarkjs", "pytest"'
     required: true
   output-file-path:
     description: 'A path to file which contains the benchmark output'

config.ts

@@ -3,7 +3,7 @@ import { promises as fs, Stats } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
 
-export type ToolType = 'cargo' | 'go' | 'benchmarkjs';
+export type ToolType = 'cargo' | 'go' | 'benchmarkjs' | 'pytest';
 export interface Config {
     name: string;
     tool: ToolType;
@@ -14,7 +14,7 @@ export interface Config {
     autoPush: boolean;
 }
 
-const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs'];
+const VALID_TOOLS: ToolType[] = ['cargo', 'go', 'benchmarkjs', 'pytest'];
 
 function validateToolType(tool: string): asserts tool is ToolType {
     if ((VALID_TOOLS as string[]).includes(tool)) {

examples/pytest/.gitignore

@@ -0,0 +1,4 @@
+/venv
+/__pycache__
+/.benchmarks
+/.pytest_cache

examples/pytest/bench.py

@@ -0,0 +1,8 @@
+from fib import fib
+import pytest
+
+def test_fib_10(benchmark):
+    benchmark(fib, 10)
+
+def test_fib_20(benchmark):
+    benchmark(fib, 20)

examples/pytest/fib/__init__.py

@@ -0,0 +1,4 @@
+def fib(n):
+    if n <= 1:
+        return 1
+    return fib(n - 2) + fib(n - 1)

examples/pytest/requirements.txt

@@ -0,0 +1,14 @@
+atomicwrites==1.3.0
+attrs==19.3.0
+importlib-metadata==0.23
+more-itertools==7.2.0
+packaging==19.2
+pluggy==0.13.0
+py==1.8.0
+py-cpuinfo==5.0.0
+pyparsing==2.4.5
+pytest==5.2.4
+pytest-benchmark==3.2.2
+six==1.13.0
+wcwidth==0.1.7
+zipp==0.6.0

examples/pytest/setup.py

@@ -0,0 +1,11 @@
+from setuptools import setup
+
+setup(
+    name='benchmark-example',
+    version='0.0.0',
+    url='https://github.com/rhysd/github-action-benchmark',
+    author='rhysd <https://rhysd.github.io>',
+    author_email='github@users.noreply.github.com',
+    description='Benchmark example with timeit package',
+    packages=['fib'],
+)

extract.ts

@@ -33,6 +33,72 @@ export interface Benchmark {
     benches: BenchmarkResult[];
 }
 
+export interface PytestBenchmarkJson {
+    machine_info: {
+        node: string;
+        processor: string;
+        machine: string;
+        python_compiler: string;
+        python_implementation: string;
+        python_implementation_version: string;
+        python_version: string;
+        python_build: string[];
+        release: string;
+        system: string;
+        cpu: {
+            vendor_id: string;
+            hardware: string;
+            brand: string;
+        };
+    };
+    commit_info: {
+        id: string;
+        time: string;
+        author_time: string;
+        dirty: boolean;
+        project: string;
+        branch: string;
+    };
+    benchmarks: Array<{
+        group: null | string;
+        name: string;
+        fullname: string;
+        params: null | string[];
+        param: null | string;
+        extra_info: object;
+        options: {
+            disable_gc: boolean;
+            time: string;
+            min_rounds: number;
+            max_time: number;
+            min_time: number;
+            warmup: boolean;
+        };
+        stats: {
+            min: number;
+            max: number;
+            mean: number;
+            stddev: number;
+            rounds: number;
+            median: number;
+            irq: number;
+            q1: number;
+            q3: number;
+            irq_outliers: number;
+            stddev_outliers: number;
+            outliers: string;
+            ld15iqr: number;
+            hd15iqr: number;
+            ops: number;
+            total: number;
+            data: number[];
+            iterations: number;
+        };
+    }>;
+    datetime: string;
+    version: string;
+}
+
 function extractCargoResult(output: string): BenchmarkResult[] {
     const lines = output.split('\n');
     const ret = [];
@@ -116,6 +182,23 @@ function extractBenchmarkJsResult(output: string): BenchmarkResult[] {
     return ret;
 }
 
+function extractPytestResult(output: string): BenchmarkResult[] {
+    try {
+        const json: PytestBenchmarkJson = JSON.parse(output);
+        return json.benchmarks.map(bench => {
+            const name = bench.fullname;
+            const value = bench.stats.mean;
+            const unit = 'sec/iter';
+            const range = `stddev: ${bench.stats.stddev}`;
+            return { name, value, unit, range };
+        });
+    } catch (err) {
+        throw new Error(
+            `Output file for 'pytest' must be JSON file generated by --benchmark-json option: ${err.message}`,
+        );
+    }
+}
+
 export async function extractResult(config: Config): Promise<Benchmark> {
     const output = await fs.readFile(config.outputFilePath, 'utf8');
     const { tool } = config;
@@ -131,6 +214,9 @@ export async function extractResult(config: Config): Promise<Benchmark> {
         case 'benchmarkjs':
             benches = extractBenchmarkJsResult(output);
             break;
+        case 'pytest':
+            benches = extractPytestResult(output);
+            break;
         default:
             throw new Error(`FATAL: Unexpected tool: '${tool}'`);
     }