From bd7d371a4913a51c1eaa5fafac6ace9862583c57 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Thu, 30 Jan 2025 20:28:00 +0100 Subject: [PATCH 1/8] extend size of iteratinos counter --- zbench.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zbench.zig b/zbench.zig index 48f40c0..395407f 100644 --- a/zbench.zig +++ b/zbench.zig @@ -31,11 +31,11 @@ pub const Hooks = struct { pub const Config = struct { /// Number of iterations the benchmark has been run. Initialized to 0. /// If 0 then zBench will calculate an value. - iterations: u16 = 0, + iterations: u32 = 0, /// Maximum number of iterations the benchmark can run. Default is 16384. /// This limit helps to avoid excessively long benchmark runs. - max_iterations: u16 = 16384, + max_iterations: u32 = 16384, /// Time budget for the benchmark in nanoseconds. Default is 2e9 (2 seconds). /// This value is used to determine how long a single benchmark should be allowed to run From db99e44bedded08e5c092d36d26c90a132a073c7 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Thu, 30 Jan 2025 20:33:05 +0100 Subject: [PATCH 2/8] extend size of iterations #2 --- util/runner.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/runner.zig b/util/runner.zig index 9939a95..4b77811 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -58,8 +58,8 @@ state: State, pub fn init( allocator: std.mem.Allocator, - iterations: u16, - max_iterations: u16, + iterations: u32, + max_iterations: u32, time_budget_ns: u64, track_allocations: bool, ) Error!Runner { From 27ddfc984e563c8d23f1f534610a6886807440f2 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Thu, 30 Jan 2025 20:35:24 +0100 Subject: [PATCH 3/8] extend size of iterations #3 --- util/runner.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/runner.zig b/util/runner.zig index 4b77811..ccbe4d6 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -21,7 +21,7 @@ const State = union(enum) { /// Maximum number of iterations the benchmark can run. This limit helps /// to avoid excessively long benchmark runs. - max_iterations: u16, + max_iterations: u32, /// Time budget for the benchmark in nanoseconds. This value is used to /// determine how long a single benchmark should be allowed to run From 19e995f1b15eb0676da614f4c54bccfebc308d36 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Tue, 4 Feb 2025 17:35:04 +0100 Subject: [PATCH 4/8] revise handling of maximum number of benchmark iterations --- util/runner.zig | 9 +++++---- zbench.zig | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/util/runner.zig b/util/runner.zig index ccbe4d6..c91f82f 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -11,6 +11,8 @@ pub const Readings = @import("./runner/types.zig").Readings; const Runner = @This(); +pub const MAX_N_ITER = 100_000; + const State = union(enum) { preparing: Preparing, running: Running, @@ -86,14 +88,13 @@ pub fn init( } pub fn next(self: *Runner, reading: Reading) Error!?Step { - const MAX_N = 65536; switch (self.state) { .preparing => |*st| { if (st.elapsed_ns < st.time_budget_ns and st.iteration_loops < st.max_iterations) { st.elapsed_ns += reading.timing_ns; if (st.iterations_remaining == 0) { // double N for next iteration - st.N = @min(st.N * 2, MAX_N); + st.N = @min(st.N * 2, MAX_N_ITER); st.iterations_remaining = st.N - 1; st.iteration_loops += 1; } else { @@ -103,10 +104,10 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { // Safety first: make sure the recorded durations aren't all-zero if (st.elapsed_ns == 0) st.elapsed_ns = 1; // Adjust N based on the actual duration achieved - var N: usize = @intCast((st.N * st.time_budget_ns) / st.elapsed_ns); + var N: usize = @intCast((st.N * st.time_budget_ns) / st.elapsed_ns + 1); // check that N doesn't go out of bounds if (N == 0) N = 1; - if (N > MAX_N) N = MAX_N; + if (N > MAX_N_ITER) N = MAX_N_ITER; // Now run the benchmark with the adjusted N value self.state = .{ .running = .{ .iterations_count = N, diff --git a/zbench.zig b/zbench.zig index 395407f..733e32b 100644 --- a/zbench.zig +++ b/zbench.zig @@ -35,7 +35,7 @@ pub const Config = struct { /// Maximum number of iterations the benchmark can run. Default is 16384. /// This limit helps to avoid excessively long benchmark runs. - max_iterations: u32 = 16384, + max_iterations: u32 = Runner.MAX_N_ITER, /// Time budget for the benchmark in nanoseconds. Default is 2e9 (2 seconds). /// This value is used to determine how long a single benchmark should be allowed to run From 6087419e82387875b2d8899907cfe3acb7bfa1f8 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Wed, 5 Feb 2025 20:19:10 +0100 Subject: [PATCH 5/8] revise Config field docs, make time budget a const from Runner --- zbench.zig | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/zbench.zig b/zbench.zig index 733e32b..c122db1 100644 --- a/zbench.zig +++ b/zbench.zig @@ -29,18 +29,18 @@ pub const Hooks = struct { /// Configuration for benchmarking. /// This struct holds settings to control the behavior of benchmark executions. pub const Config = struct { - /// Number of iterations the benchmark has been run. Initialized to 0. - /// If 0 then zBench will calculate an value. + /// Number of iterations for a given benchmark. + /// The default is 0, meaning 'determined automatically'. + /// Provide a specific number to override automatic determination. iterations: u32 = 0, - /// Maximum number of iterations the benchmark can run. Default is 16384. - /// This limit helps to avoid excessively long benchmark runs. - max_iterations: u32 = Runner.MAX_N_ITER, + /// Maximum number of iterations the benchmark should run. + /// A custom value for .iterations will override this property. + max_iterations: u32 = Runner.DEFAULT_MAX_N_ITER, - /// Time budget for the benchmark in nanoseconds. Default is 2e9 (2 seconds). - /// This value is used to determine how long a single benchmark should be allowed to run - /// before concluding. Helps in avoiding long-running benchmarks. - time_budget_ns: u64 = 2e9, + /// Time budget for the benchmark in nanoseconds. + /// This value is used to determine how long a benchmark run should take. + time_budget_ns: u64 = Runner.DEFAULT_TIME_BUDGET_NS, /// Configuration for lifecycle hooks in benchmarking. /// Provides the ability to define custom actions at different stages of the benchmark process: From 7a6aa6a73716f6afcc59284ab53a065c6c52d25a Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Wed, 5 Feb 2025 20:20:18 +0100 Subject: [PATCH 6/8] revise spin-up run, add test for max n iterations --- util/runner.zig | 78 +++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/util/runner.zig b/util/runner.zig index c91f82f..8f0567d 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -11,7 +11,8 @@ pub const Readings = @import("./runner/types.zig").Readings; const Runner = @This(); -pub const MAX_N_ITER = 100_000; +pub const DEFAULT_MAX_N_ITER = 100_000; +pub const DEFAULT_TIME_BUDGET_NS = 2_000_000_000; const State = union(enum) { preparing: Preparing, @@ -90,13 +91,13 @@ pub fn init( pub fn next(self: *Runner, reading: Reading) Error!?Step { switch (self.state) { .preparing => |*st| { - if (st.elapsed_ns < st.time_budget_ns and st.iteration_loops < st.max_iterations) { - st.elapsed_ns += reading.timing_ns; + st.elapsed_ns += reading.timing_ns; + st.iteration_loops += 1; + if (st.elapsed_ns <= st.time_budget_ns and st.iteration_loops < st.max_iterations) { if (st.iterations_remaining == 0) { - // double N for next iteration - st.N = @min(st.N * 2, MAX_N_ITER); + // double N for next iteration or use max_iterations + st.N = @min(st.N * 2, DEFAULT_MAX_N_ITER); st.iterations_remaining = st.N - 1; - st.iteration_loops += 1; } else { st.iterations_remaining -= 1; } @@ -104,10 +105,10 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { // Safety first: make sure the recorded durations aren't all-zero if (st.elapsed_ns == 0) st.elapsed_ns = 1; // Adjust N based on the actual duration achieved - var N: usize = @intCast((st.N * st.time_budget_ns) / st.elapsed_ns + 1); + var N: usize = st.N; // check that N doesn't go out of bounds if (N == 0) N = 1; - if (N > MAX_N_ITER) N = MAX_N_ITER; + if (N > DEFAULT_MAX_N_ITER) N = DEFAULT_MAX_N_ITER; // Now run the benchmark with the adjusted N value self.state = .{ .running = .{ .iterations_count = N, @@ -122,7 +123,7 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { return .more; }, .running => |*st| { - if (0 < st.iterations_remaining) { + if (st.iterations_remaining > 0) { const i = st.readings.iterations - st.iterations_remaining; st.readings.set(i, reading); st.iterations_remaining -= 1; @@ -172,22 +173,47 @@ pub fn status(self: Runner) Status { }; } -test "Runner" { - var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, false); +test "runner, time budget" { + var r = try Runner.init(std.testing.allocator, 0, DEFAULT_MAX_N_ITER, DEFAULT_TIME_BUDGET_NS, false); { errdefer r.abort(); + // run 10 steps spin-up until time budget is depleted => closest N is 8 + var i: usize = 0; + while (i < 10) : (i += 1) { + try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10) + 1, null))); + } + + // 7 runs yield .more as the next step try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); + try expectEq(Step.more, try r.next(Reading.init(400_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); - try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); - try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); + // number 8 is the final step + try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null))); + } + const result = try r.finish(); + defer result.deinit(); + + try expectEqSlices(u64, &.{ + 100_000_000, 200_000_000, 300_000_000, 400_000_000, + 100_000_000, 200_000_000, 300_000_000, 400_000_000, + }, result.timings_ns); +} + +test "runner, max n runs" { + var r = try Runner.init(std.testing.allocator, 0, 10, DEFAULT_TIME_BUDGET_NS, false); + { + errdefer r.abort(); + // spin-up: + var i: usize = 0; + while (i < 10) : (i += 1) { + try expectEq(Step.more, try r.next(Reading.init(1, null))); + } + // same as for the time budget: 7 runs yield .more as the next step: try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); @@ -195,31 +221,21 @@ test "Runner" { try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); + // number 8 is the final step: try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null))); } const result = try r.finish(); defer result.deinit(); - try expectEqSlices(u64, &.{ - 100_000_000, 200_000_000, 300_000_000, 400_000_000, - 100_000_000, 200_000_000, 300_000_000, 400_000_000, - }, result.timings_ns); } test "Runner - memory tracking" { var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9, true); { errdefer r.abort(); - try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 256, .count = 1 }))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 256, .count = 1 }))); - try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 512, .count = 2 }))); - try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 512, .count = 2 }))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 1024, .count = 4 }))); - try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 1024, .count = 4 }))); - try expectEq(Step.more, try r.next(Reading.init(100_000_000, .{ .max = 2048, .count = 8 }))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 2045, .count = 8 }))); - try expectEq(Step.more, try r.next(Reading.init(300_000_000, .{ .max = 4096, .count = 16 }))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 4096, .count = 16 }))); - try expectEq(Step.more, try r.next(Reading.init(200_000_000, .{ .max = 8192, .count = 32 }))); + var i: usize = 0; + while (i < 10) : (i += 1) { + try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10) + 1, null))); + } try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 1, .count = 2 }))); try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 2, .count = 4 }))); From f076308016c0d54a88df4a51a22a18f478fb0297 Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Wed, 5 Feb 2025 20:48:07 +0100 Subject: [PATCH 7/8] revise auto-determination of N runs, pt.2 --- util/runner.zig | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/util/runner.zig b/util/runner.zig index 8f0567d..4832b09 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -93,7 +93,7 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { .preparing => |*st| { st.elapsed_ns += reading.timing_ns; st.iteration_loops += 1; - if (st.elapsed_ns <= st.time_budget_ns and st.iteration_loops < st.max_iterations) { + if (st.elapsed_ns < st.time_budget_ns and st.iteration_loops < st.max_iterations) { if (st.iterations_remaining == 0) { // double N for next iteration or use max_iterations st.N = @min(st.N * 2, DEFAULT_MAX_N_ITER); @@ -105,7 +105,8 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { // Safety first: make sure the recorded durations aren't all-zero if (st.elapsed_ns == 0) st.elapsed_ns = 1; // Adjust N based on the actual duration achieved - var N: usize = st.N; + //var N: usize = @intCast((st.N * st.time_budget_ns) / st.elapsed_ns); + var N: usize = @intCast((st.iteration_loops * st.time_budget_ns) / st.elapsed_ns); // check that N doesn't go out of bounds if (N == 0) N = 1; if (N > DEFAULT_MAX_N_ITER) N = DEFAULT_MAX_N_ITER; @@ -177,13 +178,15 @@ test "runner, time budget" { var r = try Runner.init(std.testing.allocator, 0, DEFAULT_MAX_N_ITER, DEFAULT_TIME_BUDGET_NS, false); { errdefer r.abort(); - // run 10 steps spin-up until time budget is depleted => closest N is 8 + // run 10 steps spin-up until time budget is depleted => N is 10. var i: usize = 0; while (i < 10) : (i += 1) { - try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10) + 1, null))); + try expectEq(Step.more, try r.next(Reading.init(DEFAULT_TIME_BUDGET_NS / 10, null))); } - // 7 runs yield .more as the next step + // 9 runs yield .more as the next step + try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); + try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); @@ -191,14 +194,14 @@ test "runner, time budget" { try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); - // number 8 is the final step + // number 10 is the final step try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null))); } const result = try r.finish(); defer result.deinit(); try expectEqSlices(u64, &.{ - 100_000_000, 200_000_000, 300_000_000, 400_000_000, + 100_000_000, 100_000_000, 100_000_000, 200_000_000, 300_000_000, 400_000_000, 100_000_000, 200_000_000, 300_000_000, 400_000_000, }, result.timings_ns); } @@ -207,13 +210,15 @@ test "runner, max n runs" { var r = try Runner.init(std.testing.allocator, 0, 10, DEFAULT_TIME_BUDGET_NS, false); { errdefer r.abort(); - // spin-up: + // spin-up: make sure we do not exceed the time budget... var i: usize = 0; while (i < 10) : (i += 1) { - try expectEq(Step.more, try r.next(Reading.init(1, null))); + try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10) - 1, null))); } - // same as for the time budget: 7 runs yield .more as the next step: + // same as for the time budget: 9 runs yield .more as the next step: + try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); + try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); @@ -221,7 +226,7 @@ test "runner, max n runs" { try expectEq(Step.more, try r.next(Reading.init(100_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(200_000_000, null))); try expectEq(Step.more, try r.next(Reading.init(300_000_000, null))); - // number 8 is the final step: + // number 10 is the final step: try expectEq(@as(?Step, null), try r.next(Reading.init(400_000_000, null))); } const result = try r.finish(); @@ -234,7 +239,7 @@ test "Runner - memory tracking" { errdefer r.abort(); var i: usize = 0; while (i < 10) : (i += 1) { - try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10) + 1, null))); + try expectEq(Step.more, try r.next(Reading.init((DEFAULT_TIME_BUDGET_NS / 10), null))); } try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 1, .count = 2 }))); @@ -244,21 +249,23 @@ test "Runner - memory tracking" { try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 16, .count = 32 }))); try expectEq(Step.more, try r.next(Reading.init(200, .{ .max = 32, .count = 64 }))); try expectEq(Step.more, try r.next(Reading.init(300, .{ .max = 64, .count = 128 }))); - try expectEq(@as(?Step, null), try r.next(Reading.init(400, .{ .max = 128, .count = 256 }))); + try expectEq(Step.more, try r.next(Reading.init(400, .{ .max = 128, .count = 256 }))); + try expectEq(Step.more, try r.next(Reading.init(100, .{ .max = 256, .count = 512 }))); + try expectEq(@as(?Step, null), try r.next(Reading.init(200, .{ .max = 512, .count = 1024 }))); } const result = try r.finish(); defer result.deinit(); try expectEqSlices(u64, &.{ - 100, 200, 300, 400, 100, 200, 300, 400, + 100, 200, 300, 400, 100, 200, 300, 400, 100, 200, }, result.timings_ns); try expectEqSlices( usize, - &.{ 1, 2, 4, 8, 16, 32, 64, 128 }, + &.{ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512 }, result.allocations.?.maxes, ); try expectEqSlices( usize, - &.{ 2, 4, 8, 16, 32, 64, 128, 256 }, + &.{ 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 }, result.allocations.?.counts, ); } From 1fbb73ec32acd1ee9c77ce173636e856d6ec9c7b Mon Sep 17 00:00:00 2001 From: FObersteiner Date: Thu, 6 Feb 2025 07:05:40 +0100 Subject: [PATCH 8/8] further simplify spin-up phase --- util/runner.zig | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/util/runner.zig b/util/runner.zig index 4832b09..781aba8 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -93,15 +93,7 @@ pub fn next(self: *Runner, reading: Reading) Error!?Step { .preparing => |*st| { st.elapsed_ns += reading.timing_ns; st.iteration_loops += 1; - if (st.elapsed_ns < st.time_budget_ns and st.iteration_loops < st.max_iterations) { - if (st.iterations_remaining == 0) { - // double N for next iteration or use max_iterations - st.N = @min(st.N * 2, DEFAULT_MAX_N_ITER); - st.iterations_remaining = st.N - 1; - } else { - st.iterations_remaining -= 1; - } - } else { + if (st.elapsed_ns >= st.time_budget_ns or st.iteration_loops >= st.max_iterations) { // Safety first: make sure the recorded durations aren't all-zero if (st.elapsed_ns == 0) st.elapsed_ns = 1; // Adjust N based on the actual duration achieved