feat!: add warmup runs

CodSpeedHQ · Jul 25, 2023 · 9b955a7 · 9b955a7
1 parent f9fbacd
commit 9b955a7
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 4 deletions.
diff --git a/crates/bencher_compat/src/compat.rs b/crates/bencher_compat/src/compat.rs
@@ -1,4 +1,4 @@
-use codspeed::codspeed::{black_box, CodSpeed};
+use codspeed::codspeed::{black_box, CodSpeed, WARMUP_RUNS};
 
 pub struct Bencher {
     pub bytes: u64,
@@ -24,6 +24,9 @@ impl Bencher {
         F: FnMut() -> T,
     {
         let uri = self.current_uri.as_str();
+        for _ in 0..WARMUP_RUNS {
+            black_box(inner());
+        }
         self.codspeed.start_benchmark(uri);
         black_box(inner());
         self.codspeed.end_benchmark();

diff --git a/crates/codspeed/src/codspeed.rs b/crates/codspeed/src/codspeed.rs
@@ -4,6 +4,8 @@ use colored::Colorize;
 
 use crate::measurement;
 
+pub const WARMUP_RUNS: u32 = 5;
+
 //TODO: use std::hint::black_box when it's stable
 pub fn black_box<T>(dummy: T) -> T {
     unsafe {

diff --git a/crates/criterion_compat/src/compat/bencher.rs b/crates/criterion_compat/src/compat/bencher.rs
@@ -25,6 +25,9 @@ impl Bencher {
         R: FnMut() -> O,
     {
         let mut codspeed = self.codspeed.borrow_mut();
+        for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+            black_box(routine());
+        }
         codspeed.start_benchmark(self.uri.as_str());
         black_box(routine());
         codspeed.end_benchmark();
@@ -49,6 +52,11 @@ impl Bencher {
         R: FnMut(I) -> O,
     {
         let mut codspeed = self.codspeed.borrow_mut();
+        for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+            let input = black_box(setup());
+            let output = routine(input);
+            drop(black_box(output));
+        }
         let input = black_box(setup());
         codspeed.start_benchmark(self.uri.as_str());
         let output = routine(input);
@@ -87,8 +95,15 @@ impl Bencher {
         R: FnMut(&mut I) -> O,
     {
         let mut codspeed = self.codspeed.borrow_mut();
-        let mut input = black_box(setup());
 
+        for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+            let mut input = black_box(setup());
+            let output = routine(&mut input);
+            drop(black_box(output));
+            drop(black_box(input));
+        }
+
+        let mut input = black_box(setup());
         codspeed.start_benchmark(self.uri.as_str());
         let output = routine(&mut input);
         codspeed.end_benchmark();
@@ -121,6 +136,11 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> {
         let AsyncBencher { b, runner } = self;
         runner.block_on(async {
             let mut codspeed = b.codspeed.borrow_mut();
+
+            for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+                black_box(routine().await);
+            }
+
             codspeed.start_benchmark(b.uri.as_str());
             black_box(routine().await);
             codspeed.end_benchmark();
@@ -180,6 +200,13 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> {
         let AsyncBencher { b, runner } = self;
         runner.block_on(async {
             let mut codspeed = b.codspeed.borrow_mut();
+
+            for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+                let input = black_box(setup());
+                let output = routine(input).await;
+                drop(black_box(output));
+            }
+
             let input = black_box(setup());
             codspeed.start_benchmark(b.uri.as_str());
             let output = routine(input).await;
@@ -203,12 +230,18 @@ impl<'b, A: AsyncExecutor> AsyncBencher<'b, A> {
         let AsyncBencher { b, runner } = self;
         runner.block_on(async {
             let mut codspeed = b.codspeed.borrow_mut();
-            let mut input = black_box(setup());
 
+            for _ in 0..codspeed::codspeed::WARMUP_RUNS {
+                let mut input = black_box(setup());
+                let output = routine(&mut input).await;
+                drop(black_box(output));
+                drop(black_box(input));
+            }
+
+            let mut input = black_box(setup());
             codspeed.start_benchmark(b.uri.as_str());
             let output = routine(&mut input).await;
             codspeed.end_benchmark();
-
             drop(black_box(output));
             drop(black_box(input));
         });