Use stddev from benchmark to do a statistical test (#52)

test/benchmark.jl

@@ -51,20 +51,40 @@ const TEST_BENCHMARK = true
 
         @test f1() == f2() || f1() ≈ f2()
 
+        bench1 = @benchmarkable $f1()
+        bench2 = @benchmarkable $f2()
+
+        # we tune the first benchmark, and use the same tuned parameters for the second benchmark, to make the comparison fair.
+        tune!(bench1)
+        bench2.params = bench1.params
+
+        b1 = run(bench1)
+        b2 = run(bench2)
 
-        t1 = @belapsed $f1() seconds=1
-        t2 = @belapsed $f2() seconds=1
-
+        t1 = mean(b1).time
+        t2 = mean(b2).time
+
+        σ1 = BenchmarkTools.std(b1).time
+        σ2 = BenchmarkTools.std(b2).time
+
+        tdiff = t1 - t2
+        σdiff = sqrt(σ1^2 + σ2^2)
+
         ratio = t1 / t2
 
-        @info "Expr $i" ratio t1 t2 
+        @info "Expr $i" t1 t2 ratio diff=Text("$round(Int64, tdiff) ± $round(Int64, σdiff))") b1 b2
 
         if TEST_BENCHMARK
-            @test ratio < 1.2
+            # we should be faster, i.e.
+            # @test tdiff < 0
+
+            # and we have an admissible error of 2.5%
+            @test tdiff < 2*σdiff
         end
     end
 
     m.stop(w)
+    Distributed.rmprocs(p; waitfor=30)
 end
 
 
@@ -74,16 +94,15 @@ end
         w = m.Worker()
         @assert(2 == m.remotecall_fetch(+, w, 1, 1))
         m.stop(w)
-        isdefined(m, :_wait_for_exit) || return
-        m._wait_for_exit(w)
     end
 
     function launch_with_distributed()
         p = Distributed.addprocs(1) |> only
         @assert(2 == Distributed.remotecall_fetch(+, p, 1, 1))
-        Distributed.rmprocs(p) |> wait
+        Distributed.rmprocs(p; waitfor=30)
     end
 
+    # run once to precompile
     launch_with_malt()
     launch_with_distributed()