From 9b955a7c7446c12edc60ba68e5b937057d2c9254 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Fri, 30 Jun 2023 21:15:03 +0200 Subject: [PATCH 1/3] feat!: add warmup runs --- crates/bencher_compat/src/compat.rs | 5 ++- crates/codspeed/src/codspeed.rs | 2 + crates/criterion_compat/src/compat/bencher.rs | 39 +++++++++++++++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/crates/bencher_compat/src/compat.rs b/crates/bencher_compat/src/compat.rs index c0d221a..9cbd8aa 100644 --- a/crates/bencher_compat/src/compat.rs +++ b/crates/bencher_compat/src/compat.rs @@ -1,4 +1,4 @@ -use codspeed::codspeed::{black_box, CodSpeed}; +use codspeed::codspeed::{black_box, CodSpeed, WARMUP_RUNS}; pub struct Bencher { pub bytes: u64, @@ -24,6 +24,9 @@ impl Bencher { F: FnMut() -> T, { let uri = self.current_uri.as_str(); + for _ in 0..WARMUP_RUNS { + black_box(inner()); + } self.codspeed.start_benchmark(uri); black_box(inner()); self.codspeed.end_benchmark(); diff --git a/crates/codspeed/src/codspeed.rs b/crates/codspeed/src/codspeed.rs index 6543113..78a9ac5 100644 --- a/crates/codspeed/src/codspeed.rs +++ b/crates/codspeed/src/codspeed.rs @@ -4,6 +4,8 @@ use colored::Colorize; use crate::measurement; +pub const WARMUP_RUNS: u32 = 5; + //TODO: use std::hint::black_box when it's stable pub fn black_box(dummy: T) -> T { unsafe { diff --git a/crates/criterion_compat/src/compat/bencher.rs b/crates/criterion_compat/src/compat/bencher.rs index 7fb75ea..9314889 100644 --- a/crates/criterion_compat/src/compat/bencher.rs +++ b/crates/criterion_compat/src/compat/bencher.rs @@ -25,6 +25,9 @@ impl Bencher { R: FnMut() -> O, { let mut codspeed = self.codspeed.borrow_mut(); + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + black_box(routine()); + } codspeed.start_benchmark(self.uri.as_str()); black_box(routine()); codspeed.end_benchmark(); @@ -49,6 +52,11 @@ impl Bencher { R: FnMut(I) -> O, { let mut codspeed = self.codspeed.borrow_mut(); + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + let input = black_box(setup()); + let output = routine(input); + drop(black_box(output)); + } let input = black_box(setup()); codspeed.start_benchmark(self.uri.as_str()); let output = routine(input); @@ -87,8 +95,15 @@ impl Bencher { R: FnMut(&mut I) -> O, { let mut codspeed = self.codspeed.borrow_mut(); - let mut input = black_box(setup()); + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + let mut input = black_box(setup()); + let output = routine(&mut input); + drop(black_box(output)); + drop(black_box(input)); + } + + let mut input = black_box(setup()); codspeed.start_benchmark(self.uri.as_str()); let output = routine(&mut input); codspeed.end_benchmark(); @@ -121,6 +136,11 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { let AsyncBencher { b, runner } = self; runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); + + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + black_box(routine().await); + } + codspeed.start_benchmark(b.uri.as_str()); black_box(routine().await); codspeed.end_benchmark(); @@ -180,6 +200,13 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { let AsyncBencher { b, runner } = self; runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); + + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + let input = black_box(setup()); + let output = routine(input).await; + drop(black_box(output)); + } + let input = black_box(setup()); codspeed.start_benchmark(b.uri.as_str()); let output = routine(input).await; @@ -203,12 +230,18 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { let AsyncBencher { b, runner } = self; runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); - let mut input = black_box(setup()); + for _ in 0..codspeed::codspeed::WARMUP_RUNS { + let mut input = black_box(setup()); + let output = routine(&mut input).await; + drop(black_box(output)); + drop(black_box(input)); + } + + let mut input = black_box(setup()); codspeed.start_benchmark(b.uri.as_str()); let output = routine(&mut input).await; codspeed.end_benchmark(); - drop(black_box(output)); drop(black_box(input)); }); From ac7e20850766bf1fd6e714420da65bcf206307b0 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Tue, 25 Jul 2023 19:41:57 +0200 Subject: [PATCH 2/3] fix: add more opaque black boxes in the tests --- .../benches/criterion_integration/compare_functions.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/criterion_compat/benches/criterion_integration/compare_functions.rs b/crates/criterion_compat/benches/criterion_integration/compare_functions.rs index 5c9bec4..7a2563b 100644 --- a/crates/criterion_compat/benches/criterion_integration/compare_functions.rs +++ b/crates/criterion_compat/benches/criterion_integration/compare_functions.rs @@ -1,4 +1,4 @@ -use codspeed_criterion_compat::{criterion_group, BenchmarkId, Criterion}; +use codspeed_criterion_compat::{black_box, criterion_group, BenchmarkId, Criterion}; fn fibonacci_slow(n: u64) -> u64 { match n { @@ -27,14 +27,16 @@ fn fibonacci_fast(n: u64) -> u64 { fn compare_fibonaccis(c: &mut Criterion) { let mut group = c.benchmark_group("Fibonacci"); - group.bench_with_input("Recursive", &20, |b, i| b.iter(|| fibonacci_slow(*i))); + group.bench_with_input("Recursive", &20, |b, i| { + b.iter(|| fibonacci_slow(black_box(*i))) + }); group.bench_with_input("Iterative", &20, |b, i| b.iter(|| fibonacci_fast(*i))); } fn compare_fibonaccis_group(c: &mut Criterion) { let mut group = c.benchmark_group("Fibonacci3"); for i in 20..=21 { group.bench_with_input(BenchmarkId::new("Recursive", i), &i, |b, i| { - b.iter(|| fibonacci_slow(*i)) + b.iter(|| fibonacci_slow(black_box(*i))) }); group.bench_with_input(BenchmarkId::new("Iterative", i), &i, |b, i| { b.iter(|| fibonacci_fast(*i)) From bb6087f5e26c337c1f03e60d3ced05eb76883af5 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Wed, 26 Jul 2023 11:19:51 +0200 Subject: [PATCH 3/3] feat: harden criterion compat library against optimization --- crates/criterion_compat/src/compat/bencher.rs | 103 ++++++++++-------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/crates/criterion_compat/src/compat/bencher.rs b/crates/criterion_compat/src/compat/bencher.rs index 9314889..53568d9 100644 --- a/crates/criterion_compat/src/compat/bencher.rs +++ b/crates/criterion_compat/src/compat/bencher.rs @@ -25,12 +25,17 @@ impl Bencher { R: FnMut() -> O, { let mut codspeed = self.codspeed.borrow_mut(); - for _ in 0..codspeed::codspeed::WARMUP_RUNS { - black_box(routine()); + // NOTE: this structure hardens our benchmark against dead code elimination + // https://godbolt.org/z/KnYeKMd1o + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine()); + } else { + codspeed.start_benchmark(self.uri.as_str()); + black_box(routine()); + codspeed.end_benchmark(); + } } - codspeed.start_benchmark(self.uri.as_str()); - black_box(routine()); - codspeed.end_benchmark(); } #[inline(never)] @@ -52,17 +57,20 @@ impl Bencher { R: FnMut(I) -> O, { let mut codspeed = self.codspeed.borrow_mut(); - for _ in 0..codspeed::codspeed::WARMUP_RUNS { + + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { let input = black_box(setup()); - let output = routine(input); + let output = if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine(input)) + } else { + let input = black_box(setup()); + codspeed.start_benchmark(self.uri.as_str()); + let output = black_box(routine(input)); + codspeed.end_benchmark(); + output + }; drop(black_box(output)); } - let input = black_box(setup()); - codspeed.start_benchmark(self.uri.as_str()); - let output = routine(input); - codspeed.end_benchmark(); - - drop(black_box(output)); } pub fn iter_with_setup(&mut self, setup: S, routine: R) @@ -96,20 +104,19 @@ impl Bencher { { let mut codspeed = self.codspeed.borrow_mut(); - for _ in 0..codspeed::codspeed::WARMUP_RUNS { + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { let mut input = black_box(setup()); - let output = routine(&mut input); + let output = if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine(&mut input)) + } else { + codspeed.start_benchmark(self.uri.as_str()); + let output = black_box(routine(&mut input)); + codspeed.end_benchmark(); + output + }; drop(black_box(output)); drop(black_box(input)); } - - let mut input = black_box(setup()); - codspeed.start_benchmark(self.uri.as_str()); - let output = routine(&mut input); - codspeed.end_benchmark(); - - drop(black_box(output)); - drop(black_box(input)); } #[cfg(feature = "async")] @@ -136,14 +143,15 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { let AsyncBencher { b, runner } = self; runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); - - for _ in 0..codspeed::codspeed::WARMUP_RUNS { - black_box(routine().await); + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { + if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine().await); + } else { + codspeed.start_benchmark(b.uri.as_str()); + black_box(routine().await); + codspeed.end_benchmark(); + } } - - codspeed.start_benchmark(b.uri.as_str()); - black_box(routine().await); - codspeed.end_benchmark(); }); } @@ -201,17 +209,18 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); - for _ in 0..codspeed::codspeed::WARMUP_RUNS { + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { let input = black_box(setup()); - let output = routine(input).await; + let output = if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine(input).await) + } else { + codspeed.start_benchmark(b.uri.as_str()); + let output = black_box(routine(input).await); + codspeed.end_benchmark(); + output + }; drop(black_box(output)); } - - let input = black_box(setup()); - codspeed.start_benchmark(b.uri.as_str()); - let output = routine(input).await; - codspeed.end_benchmark(); - drop(black_box(output)); }) } @@ -231,19 +240,19 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> { runner.block_on(async { let mut codspeed = b.codspeed.borrow_mut(); - for _ in 0..codspeed::codspeed::WARMUP_RUNS { + for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 { let mut input = black_box(setup()); - let output = routine(&mut input).await; + let output = if i < codspeed::codspeed::WARMUP_RUNS { + black_box(routine(&mut input).await) + } else { + codspeed.start_benchmark(b.uri.as_str()); + let output = black_box(routine(&mut input).await); + codspeed.end_benchmark(); + output + }; drop(black_box(output)); drop(black_box(input)); } - - let mut input = black_box(setup()); - codspeed.start_benchmark(b.uri.as_str()); - let output = routine(&mut input).await; - codspeed.end_benchmark(); - drop(black_box(output)); - drop(black_box(input)); }); } }