diff --git a/examples/default.rb b/examples/default.rb new file mode 100644 index 0000000..deb370b --- /dev/null +++ b/examples/default.rb @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby + +require 'benchmark/ips' + +Benchmark.ips do |x| + # Typical mode, runs the block as many times as it can + x.report("addition") { 1 + 2 } + + # To reduce overhead, the number of iterations is passed in + # and the block must run the code the specific number of times. + # Used for when the workload is very small and any overhead + # introduces incorrectable errors. + x.report(:addition2) do |times| + i = 0 + while i < times + 1 + 2 + i += 1 + end + end + + # To reduce overhead even more, grafts the code given into + # the loop that performs the iterations internally to reduce + # overhead. Typically not needed, use the |times| form instead. + x.report("addition3", "1 + 2") + + # Really long labels should be formatted correctly + x.report("addition-test-long-label") { 1 + 2 } + + x.compare! +end + +puts <<-EOD +Typical results will show addition2 & addition3 to be the most performant, and +they should perform reasonably similarly. You should see addition and +addition-test-long-label to perform very similarly to each other (as they are +running the same test, just with different labels), and they should both run in +the neighborhood of 3.5 times slower than addition2 and addition3." +EOD diff --git a/lib/benchmark/ips/job.rb b/lib/benchmark/ips/job.rb index fd00890..d049e51 100644 --- a/lib/benchmark/ips/job.rb +++ b/lib/benchmark/ips/job.rb @@ -77,7 +77,7 @@ def initialize opts={} @full_report = Report.new # Default warmup and calculation time in seconds. - @warmup = 2 + @warmup = nil @time = 5 @iterations = 1 @@ -247,11 +247,16 @@ def clear_held_results end def run - if @warmup && @warmup != 0 then - @out.start_warming - @iterations.times do - run_warmup + if @warmup + if @warmup != 0 + @out.start_warming + @iterations.times do + run_warmup + end end + else + @out.start_warming + run_auto_warmup end @out.start_running @@ -308,6 +313,82 @@ def run_warmup end end + def run_single_autowarm(item) + @out.warming item.label, warmup + + # The idea is to run the item until the cycles per 100ms timing + # is within 1% of the previous run. This means that the default is still + # 2 seconds like it was originally, but now if those 2 seconds didn't + # yield runs that were close enough together, the warmup will continue to + # run. + # + # It will run for a maximum of 30 seconds. + warmup = 1 + prev = nil + warmup_time_us = nil + + 30.times do + Timing.clean_env + + # Run for up to half of the configured warmup time with an increasing + # number of cycles to reduce overhead and improve accuracy. + # This also avoids running with a constant number of cycles, which a + # JIT might speculate on and then have to recompile in #run_benchmark. + before = Timing.now + target = Timing.add_second before, warmup + + cycles = 1 + begin + t0 = Timing.now + item.call_times cycles + t1 = Timing.now + warmup_iter = cycles + warmup_time_us = Timing.time_us(t0, t1) + + # If the number of cycles would go outside the 32-bit signed integers range + # then exit the loop to avoid overflows and start the 100ms warmup runs + break if cycles >= POW_2_30 + cycles *= 2 + end while Timing.now + warmup_time_us * 2 < target + + per = cycles_per_100ms warmup_time_us, warmup_iter + + if prev != nil + diff = if per > prev + per / prev + else + prev / per + end + + if diff - 1.0 <= 0.1 + @timing[item] = cycles + break + end + end + + prev = per + + # Run for the remaining of warmup in a similar way as #run_benchmark. + target = Timing.add_second before, warmup + while Timing.now + MICROSECONDS_PER_100MS < target + item.call_times cycles + end + end + + @out.warmup_stats warmup_time_us, @timing[item] + end + + # Run warmup. + def run_auto_warmup + @list.each do |item| + next if run_single? && @held_results && @held_results.key?(item.label) + + run_single_autowarm(item) + + break if run_single? + end + end + # Run calculation. def run_benchmark @list.each do |item|