Implement automatic warmup timing

examples/default.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require 'benchmark/ips'
+
+Benchmark.ips do |x|
+  # Typical mode, runs the block as many times as it can
+  x.report("addition") { 1 + 2 }
+
+  # To reduce overhead, the number of iterations is passed in
+  # and the block must run the code the specific number of times.
+  # Used for when the workload is very small and any overhead
+  # introduces incorrectable errors.
+  x.report(:addition2) do |times|
+    i = 0
+    while i < times
+      1 + 2
+      i += 1
+    end
+  end
+
+  # To reduce overhead even more, grafts the code given into
+  # the loop that performs the iterations internally to reduce
+  # overhead. Typically not needed, use the |times| form instead.
+  x.report("addition3", "1 + 2")
+
+  # Really long labels should be formatted correctly
+  x.report("addition-test-long-label") { 1 + 2 }
+
+  x.compare!
+end
+
+puts <<-EOD
+Typical results will show addition2 & addition3 to be the most performant, and
+they should perform reasonably similarly. You should see addition and
+addition-test-long-label to perform very similarly to each other (as they are
+running the same test, just with different labels), and they should both run in
+the neighborhood of 3.5 times slower than addition2 and addition3."
+EOD

lib/benchmark/ips/job.rb

@@ -77,7 +77,7 @@ def initialize opts={}
         @full_report = Report.new
 
         # Default warmup and calculation time in seconds.
-        @warmup = 2
+        @warmup = nil
         @time = 5
         @iterations = 1
 
@@ -247,11 +247,16 @@ def clear_held_results
       end
 
       def run
-        if @warmup && @warmup != 0 then
-          @out.start_warming
-          @iterations.times do
-            run_warmup
+        if @warmup
+          if @warmup != 0
+            @out.start_warming
+            @iterations.times do
+              run_warmup
+            end
           end
+        else
+          @out.start_warming
+          run_auto_warmup
         end
 
         @out.start_running
@@ -308,6 +313,82 @@ def run_warmup
         end
       end
 
+      def run_single_autowarm(item)
+        @out.warming item.label, warmup
+
+        # The idea is to run the item until the cycles per 100ms timing
+        # is within 1% of the previous run. This means that the default is still
+        # 2 seconds like it was originally, but now if those 2 seconds didn't
+        # yield runs that were close enough together, the warmup will continue to
+        # run.
+        #
+        # It will run for a maximum of 30 seconds.
+        warmup = 1
+        prev = nil
+        warmup_time_us = nil
+
+        30.times do
+          Timing.clean_env
+
+          # Run for up to half of the configured warmup time with an increasing
+          # number of cycles to reduce overhead and improve accuracy.
+          # This also avoids running with a constant number of cycles, which a
+          # JIT might speculate on and then have to recompile in #run_benchmark.
+          before = Timing.now
+          target = Timing.add_second before, warmup
+
+          cycles = 1
+          begin
+            t0 = Timing.now
+            item.call_times cycles
+            t1 = Timing.now
+            warmup_iter = cycles
+            warmup_time_us = Timing.time_us(t0, t1)
+
+            # If the number of cycles would go outside the 32-bit signed integers range
+            # then exit the loop to avoid overflows and start the 100ms warmup runs
+            break if cycles >= POW_2_30
+            cycles *= 2
+          end while Timing.now + warmup_time_us * 2 < target
+
+          per = cycles_per_100ms warmup_time_us, warmup_iter
+
+          if prev != nil
+            diff = if per > prev
+              per / prev
+            else
+              prev / per
+            end
+
+            if diff - 1.0 <= 0.1
+              @timing[item] = cycles
+              break
+            end
+          end
+
+          prev = per
+
+          # Run for the remaining of warmup in a similar way as #run_benchmark.
+          target = Timing.add_second before, warmup
+          while Timing.now + MICROSECONDS_PER_100MS < target
+            item.call_times cycles
+          end
+        end
+
+        @out.warmup_stats warmup_time_us, @timing[item]
+      end
+
+      # Run warmup.
+      def run_auto_warmup
+        @list.each do |item|
+          next if run_single? && @held_results && @held_results.key?(item.label)
+
+          run_single_autowarm(item)
+
+          break if run_single?
+        end
+      end
+
       # Run calculation.
       def run_benchmark
         @list.each do |item|