diff --git a/README.md b/README.md index 50c140f..f81aa83 100644 --- a/README.md +++ b/README.md @@ -105,10 +105,8 @@ You can then run your benchmarks in a test: test "bench test" { var bench = zbench.Benchmark.init(std.testing.allocator, .{}); defer bench.deinit(); - try bench.add("My Benchmark", myBenchmark, .{ .iterations = 10 }); - const results = try bench.run(); - defer results.deinit(); - try results.prettyPrint(stdout, true); + try bench.add("My Benchmark", myBenchmark, .{}); + try bench.run(std.io.getStdOut().writer()); } ``` diff --git a/build.zig b/build.zig index 8f51043..f5e1098 100644 --- a/build.zig +++ b/build.zig @@ -39,6 +39,7 @@ fn setupTesting(b: *std.Build, target: std.zig.CrossTarget, optimize: std.builti .{ .name = "optional", .path = "util/optional.zig" }, .{ .name = "platform", .path = "util/platform.zig" }, .{ .name = "runner", .path = "util/runner.zig" }, + .{ .name = "statistics", .path = "util/statistics.zig" }, .{ .name = "zbench", .path = "zbench.zig" }, }; @@ -61,6 +62,7 @@ fn setupExamples(b: *std.Build, target: std.zig.CrossTarget, optimize: std.built "basic", "bubble_sort", "hooks", + "json", "parameterised", "progress", "sleep", diff --git a/examples/basic.zig b/examples/basic.zig index 2f72817..bf7686a 100644 --- a/examples/basic.zig +++ b/examples/basic.zig @@ -15,5 +15,7 @@ test "bench test basic" { defer bench.deinit(); try bench.add("My Benchmark", myBenchmark, .{}); + + try stdout.writeAll("\n"); try bench.run(stdout); } diff --git a/examples/bubble_sort.zig b/examples/bubble_sort.zig index 833595d..93382ff 100644 --- a/examples/bubble_sort.zig +++ b/examples/bubble_sort.zig @@ -27,5 +27,6 @@ test "bench test bubbleSort" { try bench.add("Bubble Sort Benchmark", myBenchmark, .{}); + try stdout.writeAll("\n"); try bench.run(stdout); } diff --git a/examples/hooks.zig b/examples/hooks.zig index cfac6e4..7ec1edb 100644 --- a/examples/hooks.zig +++ b/examples/hooks.zig @@ -29,5 +29,7 @@ test "bench test hooks" { .after_all = afterAllHook, }, }); + + try stdout.writeAll("\n"); try bench.run(stdout); } diff --git a/examples/json.zig b/examples/json.zig new file mode 100644 index 0000000..577776d --- /dev/null +++ b/examples/json.zig @@ -0,0 +1,35 @@ +const std = @import("std"); +const zbench = @import("zbench"); +const test_allocator = std.testing.allocator; + +fn myBenchmark(alloc: std.mem.Allocator) void { + var result: usize = 0; + for (0..2_000) |i| { + std.mem.doNotOptimizeAway(i); + result += i * i; + const buf = alloc.alloc(u8, 1024) catch unreachable; + defer alloc.free(buf); + } +} + +test "bench test json" { + const stdout = std.io.getStdOut().writer(); + var bench = zbench.Benchmark.init(test_allocator, .{}); + defer bench.deinit(); + + try bench.add("My Benchmark 1", myBenchmark, .{ .iterations = 10 }); + + try stdout.writeAll("["); + var iter = try bench.iterator(); + var i: usize = 0; + while (try iter.next()) |step| switch (step) { + .progress => |_| {}, + .result => |x| { + defer x.deinit(); + defer i += 1; + if (0 < i) try stdout.writeAll(", "); + try x.writeJSON(stdout); + }, + }; + try stdout.writeAll("]\n"); +} diff --git a/examples/progress.zig b/examples/progress.zig index 96c3e86..c70ad05 100644 --- a/examples/progress.zig +++ b/examples/progress.zig @@ -31,7 +31,7 @@ test "bench test progress" { defer progress_node.end(); try stdout.writeAll("\n"); - try bench.prettyPrintHeader(stdout); + try zbench.prettyPrintHeader(stdout); var iter = try bench.iterator(); while (try iter.next()) |step| switch (step) { .progress => |p| { diff --git a/examples/systeminfo.zig b/examples/systeminfo.zig index c0e5fec..3634c7d 100644 --- a/examples/systeminfo.zig +++ b/examples/systeminfo.zig @@ -1,22 +1,7 @@ const std = @import("std"); const zbench = @import("zbench"); -fn myBenchmark(_: std.mem.Allocator) void { - var result: usize = 0; - for (0..1_000_000) |i| { - std.mem.doNotOptimizeAway(i); - result += i * i; - } -} - -test "bench test system info" { +test "system info" { const stdout = std.io.getStdOut().writer(); - var bench = zbench.Benchmark.init(std.testing.allocator, .{}); - defer bench.deinit(); - - const sysinfo = try bench.getSystemInfo(); - try std.fmt.format(stdout, "\n{}\n", .{sysinfo}); - - try bench.add("My Benchmark", myBenchmark, .{}); - try bench.run(stdout); + try stdout.print("\n\n{}\n", .{try zbench.getSystemInfo()}); } diff --git a/util/format.zig b/util/format.zig index 62b673e..6ba0f35 100644 --- a/util/format.zig +++ b/util/format.zig @@ -1,98 +1,33 @@ const std = @import("std"); -const log = std.log.scoped(.zbench_format); -pub fn memorySize(bytes: u64, allocator: std.mem.Allocator) ![]const u8 { - const units = .{ "B", "KB", "MB", "GB", "TB" }; - var size: f64 = @floatFromInt(bytes); - var unit_index: usize = 0; - - while (size >= 1024 and unit_index < units.len - 1) : (unit_index += 1) { - size /= 1024; - } - - const unit = switch (unit_index) { - 0 => "B", - 1 => "KB", - 2 => "MB", - 3 => "GB", - 4 => "TB", - 5 => "PB", - 6 => "EB", - else => unreachable, +fn FormatJSONArrayData(comptime T: type) type { + return struct { + values: []const T, + + const Self = @This(); + + fn format( + data: Self, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.writeAll("["); + for (data.values, 0..) |x, i| { + if (0 < i) try writer.writeAll(", "); + try writer.print("{}", .{x}); + } + try writer.writeAll("]"); + } }; - - // Format the result with two decimal places if needed - var buf: [64]u8 = undefined; // Buffer for formatting - const formattedSize = try std.fmt.bufPrint(&buf, "{d:.2} {s}", .{ size, unit }); - return allocator.dupe(u8, formattedSize); -} - -/// Pretty-prints the header for the result pretty-print table -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintHeader(writer: anytype) !void { - try writer.print( - "{s:<22} {s:<8} {s:<14} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10}\n", - .{ - "benchmark", - "runs", - "total time", - "time/run (avg ± σ)", - "(min ... max)", - "p75", - "p99", - "p995", - }, - ); - const dashes = "-------------------------"; - try writer.print(dashes ++ dashes ++ dashes ++ dashes ++ dashes ++ "\n", .{}); -} - -/// Pretty-prints the name of the benchmark -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintName(name: []const u8, writer: anytype) !void { - try writer.print("{s:<22} ", .{name}); -} - -/// Pretty-prints the number of total operations (or runs) of the benchmark performed -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintTotalOperations(total_operations: u64, writer: anytype) !void { - try writer.print("{d:<8} ", .{total_operations}); -} - -/// Pretty-prints the total time it took to perform all the runs -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintTotalTime(total_time: u64, writer: anytype) !void { - try writer.print("{s:<14} ", .{std.fmt.fmtDuration(total_time)}); -} - -/// Pretty-prints the average (arithmetic mean) and the standard deviation of the durations -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintAvgStd(avg: u64, stddev: u64, writer: anytype) !void { - var buffer: [128]u8 = undefined; - const str = try std.fmt.bufPrint(&buffer, "{} ± {}", .{ - std.fmt.fmtDuration(avg), - std.fmt.fmtDuration(stddev), - }); - try writer.print("{s:<22} ", .{str}); -} - -/// Pretty-prints the minumim and maximum duration -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintMinMax(min: u64, max: u64, writer: anytype) !void { - var buffer: [128]u8 = undefined; - const str = try std.fmt.bufPrint(&buffer, "({} ... {})", .{ - std.fmt.fmtDuration(min), - std.fmt.fmtDuration(max), - }); - try writer.print("{s:<28} ", .{str}); } -/// Pretty-prints the 75th, 99th and 99.5th percentile of the durations -/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) -pub fn prettyPrintPercentiles(p75: u64, p99: u64, p995: u64, writer: anytype) !void { - try writer.print("{s:<10} {s:<10} {s:<10}", .{ - std.fmt.fmtDuration(p75), - std.fmt.fmtDuration(p99), - std.fmt.fmtDuration(p995), - }); +pub fn fmtJSONArray( + comptime T: type, + values: []const T, +) std.fmt.Formatter(FormatJSONArrayData(T).format) { + const data = FormatJSONArrayData(T){ .values = values }; + return .{ .data = data }; } diff --git a/util/platform.zig b/util/platform.zig index 0506253..b5162b7 100644 --- a/util/platform.zig +++ b/util/platform.zig @@ -1,8 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); -const format = @import("format.zig"); - const lnx = @import("os/linux.zig"); const mac = @import("os/osx.zig"); const win = @import("os/windows.zig"); diff --git a/util/runner.zig b/util/runner.zig index 0ef5f1c..c3f46bf 100644 --- a/util/runner.zig +++ b/util/runner.zig @@ -4,6 +4,9 @@ const expectEqSlices = std.testing.expectEqualSlices; pub const Error = @import("./runner/types.zig").Error; pub const Step = @import("./runner/types.zig").Step; +pub const Reading = @import("./runner/types.zig").Reading; +pub const Readings = @import("./runner/types.zig").Readings; + const Runner = @This(); const State = union(enum) { @@ -55,7 +58,7 @@ pub fn init( iterations: u16, max_iterations: u16, time_budget_ns: u64, -) !Runner { +) Error!Runner { return if (iterations == 0) .{ .allocator = allocator, .state = .{ .preparing = .{ @@ -72,12 +75,12 @@ pub fn init( }; } -pub fn next(self: *Runner, ns: u64) Runner.Error!?Runner.Step { +pub fn next(self: *Runner, reading: Reading) Error!?Step { const MAX_N = 65536; switch (self.state) { .preparing => |*st| { if (st.elapsed_ns < st.time_budget_ns and st.iteration_loops < st.max_iterations) { - st.elapsed_ns += ns; + st.elapsed_ns += reading.timing_ns; if (st.iterations_remaining == 0) { // double N for next iteration st.N = @min(st.N * 2, MAX_N); @@ -105,7 +108,8 @@ pub fn next(self: *Runner, ns: u64) Runner.Error!?Runner.Step { }, .running => |*st| { if (0 < st.iterations_remaining) { - st.timings_ns[st.timings_ns.len - st.iterations_remaining] = ns; + const i = st.timings_ns.len - st.iterations_remaining; + st.timings_ns[i] = reading.timing_ns; st.iterations_remaining -= 1; } return if (st.iterations_remaining == 0) null else .more; @@ -115,10 +119,14 @@ pub fn next(self: *Runner, ns: u64) Runner.Error!?Runner.Step { /// The next() function has returned null and there are no more steps to /// complete, so get the timing results. -pub fn finish(self: *Runner) Runner.Error![]u64 { +pub fn finish(self: *Runner) Error!Readings { return switch (self.state) { - .preparing => &.{}, - .running => |st| st.timings_ns, + .preparing => .{ + .timings_ns = &.{}, + }, + .running => |st| .{ + .timings_ns = st.timings_ns, + }, }; } @@ -152,30 +160,30 @@ test "Runner" { var r = try Runner.init(std.testing.allocator, 0, 16384, 2e9); { errdefer r.abort(); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(Runner.Step.more, try r.next(200_000_000)); - try expectEq(@as(?Runner.Step, null), try r.next(200_000_000)); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(Step.more, try r.next(Reading.init(200_000_000))); + try expectEq(@as(?Step, null), try r.next(Reading.init(200_000_000))); } - const timings_ns = try r.finish(); - defer std.testing.allocator.free(timings_ns); + const result = try r.finish(); + defer std.testing.allocator.free(result.timings_ns); try expectEqSlices(u64, &.{ 200_000_000, 200_000_000, 200_000_000, 200_000_000, 200_000_000, 200_000_000, 200_000_000, 200_000_000, - }, timings_ns); + }, result.timings_ns); } diff --git a/util/runner/types.zig b/util/runner/types.zig index 02c495f..61eb7a4 100644 --- a/util/runner/types.zig +++ b/util/runner/types.zig @@ -3,3 +3,17 @@ const std = @import("std"); pub const Error = std.mem.Allocator.Error; pub const Step = enum { more }; + +pub const Reading = struct { + timing_ns: u64, + + pub fn init(timing_ns: u64) Reading { + return .{ + .timing_ns = timing_ns, + }; + } +}; + +pub const Readings = struct { + timings_ns: []u64, +}; diff --git a/util/statistics.zig b/util/statistics.zig new file mode 100644 index 0000000..b084a62 --- /dev/null +++ b/util/statistics.zig @@ -0,0 +1,169 @@ +const std = @import("std"); + +pub fn Statistics(comptime T: type) type { + return struct { + total: T, + mean: T, + stddev: T, + min: T, + max: T, + percentiles: Percentiles, + + const Self = @This(); + + pub const Percentiles = struct { + p75: T, + p99: T, + p995: T, + }; + + /// Create a statistical summary of a dataset, NB. assumes that the + /// readings are sorted. + pub fn init(readings: []const T) Self { + const len = readings.len; + + // Calculate total and mean + var total: T = 0; + for (readings) |n| total += n; + const mean: T = if (0 < len) total / len else 0; + + // Calculate standard deviation + const stddev: T = blk: { + var nvar: T = 0; + for (readings) |n| { + const sd = if (n < mean) mean - n else n - mean; + nvar += sd * sd; + } + break :blk if (1 < len) std.math.sqrt(nvar / (len - 1)) else 0; + }; + + return Self{ + .total = total, + .mean = mean, + .stddev = stddev, + .min = if (len == 0) 0 else readings[0], + .max = if (len == 0) 0 else readings[len - 1], + .percentiles = Percentiles{ + .p75 = if (len == 0) 0 else readings[len * 75 / 100], + .p99 = if (len == 0) 0 else readings[len * 99 / 100], + .p995 = if (len == 0) 0 else readings[len * 995 / 1000], + }, + }; + } + + fn formatJSON( + data: struct { []const u8, Self }, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.print( + \\{{ "units": "{s}", + \\ "total": {d}, + \\ "mean": {d}, + \\ "stddev": {d}, + \\ "min": {d}, + \\ "max": {d}, + \\ "percentiles": {{"p75": {d}, "p99": {d}, "p995": {d} }} }} + , + .{ + data[0], + data[1].total, + data[1].mean, + data[1].stddev, + data[1].min, + data[1].max, + data[1].percentiles.p75, + data[1].percentiles.p99, + data[1].percentiles.p995, + }, + ); + } + }; +} + +pub fn fmtJSON( + comptime T: type, + unit: []const u8, + stats: Statistics(T), +) std.fmt.Formatter(Statistics(T).formatJSON) { + return .{ .data = .{ unit, stats } }; +} + +test "Statistics" { + const expectEqDeep = std.testing.expectEqualDeep; + { + var timings_ns = std.ArrayList(u64).init(std.testing.allocator); + defer timings_ns.deinit(); + try expectEqDeep(Statistics(u64){ + .total = 0, + .mean = 0, + .stddev = 0, + .min = 0, + .max = 0, + .percentiles = .{ + .p75 = 0, + .p99 = 0, + .p995 = 0, + }, + }, Statistics(u64).init(timings_ns.items)); + } + + { + var timings_ns = std.ArrayList(u64).init(std.testing.allocator); + defer timings_ns.deinit(); + try timings_ns.append(1); + try expectEqDeep(Statistics(u64){ + .total = 1, + .mean = 1, + .stddev = 0, + .min = 1, + .max = 1, + .percentiles = .{ + .p75 = 1, + .p99 = 1, + .p995 = 1, + }, + }, Statistics(u64).init(timings_ns.items)); + } + + { + var timings_ns = std.ArrayList(u64).init(std.testing.allocator); + defer timings_ns.deinit(); + try timings_ns.append(1); + for (1..16) |i| try timings_ns.append(i); + try expectEqDeep(Statistics(u64){ + .total = 121, + .mean = 7, + .stddev = 4, + .min = 1, + .max = 15, + .percentiles = .{ + .p75 = 12, + .p99 = 15, + .p995 = 15, + }, + }, Statistics(u64).init(timings_ns.items)); + } + + { + var timings_ns = std.ArrayList(u64).init(std.testing.allocator); + defer timings_ns.deinit(); + try timings_ns.append(1); + for (1..101) |i| try timings_ns.append(i); + try expectEqDeep(Statistics(u64){ + .total = 5051, + .mean = 50, + .stddev = 29, + .min = 1, + .max = 100, + .percentiles = .{ + .p75 = 75, + .p99 = 99, + .p995 = 100, + }, + }, Statistics(u64).init(timings_ns.items)); + } +} diff --git a/zbench.zig b/zbench.zig index 322a1aa..19b9edb 100644 --- a/zbench.zig +++ b/zbench.zig @@ -5,6 +5,8 @@ const std = @import("std"); const expectEq = std.testing.expectEqual; +pub const statistics = @import("./util/statistics.zig"); +const Statistics = statistics.Statistics; const Color = @import("./util/color.zig").Color; const format = @import("./util/format.zig"); const Optional = @import("./util/optional.zig").Optional; @@ -14,7 +16,7 @@ const Runner = @import("./util/runner.zig"); /// Hooks containing optional hooks for lifecycle events in benchmarking. /// Each field in this struct is a nullable function pointer. -const Hooks = struct { +pub const Hooks = struct { before_all: ?*const fn () void = null, after_all: ?*const fn () void = null, before_each: ?*const fn () void = null, @@ -59,6 +61,22 @@ const Definition = struct { context: *const anyopaque, }, }, + + /// Run and time a benchmark function once, as well as running before and + /// after hooks. + fn run(self: Definition, allocator: std.mem.Allocator) !Runner.Reading { + if (self.config.hooks.before_each) |hook| hook(); + defer if (self.config.hooks.after_each) |hook| hook(); + + var t = try std.time.Timer.start(); + switch (self.defn) { + .simple => |func| func(allocator), + .parameterised => |x| x.func(@ptrCast(x.context), allocator), + } + return Runner.Reading{ + .timing_ns = t.read(), + }; + } }; /// A function pointer type that represents a benchmark function. @@ -172,8 +190,8 @@ pub const Benchmark = struct { const runner_step = blk: { errdefer self.abort(); - const ns = try self.b.runFunc(self.remaining[0]); - break :blk try runner.next(ns); + const reading = try self.remaining[0].run(self.allocator); + break :blk try runner.next(reading); }; if (runner_step) |_| { const total_benchmarks = self.b.benchmarks.items.len; @@ -224,7 +242,7 @@ pub const Benchmark = struct { const progress_node = progress.start("", 0); defer progress_node.end(); - try self.prettyPrintHeader(writer); + try prettyPrintHeader(writer); var iter = try self.iterator(); while (try iter.next()) |step| switch (step) { .progress => |p| { @@ -243,31 +261,31 @@ pub const Benchmark = struct { }, }; } +}; - /// Run and time a benchmark function once, as well as running before and - /// after hooks. - fn runFunc(self: Benchmark, defn: Definition) !u64 { - if (defn.config.hooks.before_each) |hook| hook(); - defer if (defn.config.hooks.after_each) |hook| hook(); - - var t = try std.time.Timer.start(); - switch (defn.defn) { - .simple => |func| func(self.allocator), - .parameterised => |x| x.func(@ptrCast(x.context), self.allocator), - } - return t.read(); - } - - /// Write the prettyPrint() header to a writer. - pub fn prettyPrintHeader(_: Benchmark, writer: anytype) !void { - try format.prettyPrintHeader(writer); - } +/// Write the prettyPrint() header to a writer. +pub fn prettyPrintHeader(writer: anytype) !void { + try writer.print( + "{s:<22} {s:<8} {s:<14} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10}\n", + .{ + "benchmark", + "runs", + "total time", + "time/run (avg ± σ)", + "(min ... max)", + "p75", + "p99", + "p995", + }, + ); + const dashes = "-------------------------"; + try writer.print(dashes ++ dashes ++ dashes ++ dashes ++ dashes ++ "\n", .{}); +} - /// Get a copy of the system information, cpu type, cores, memory, etc. - pub fn getSystemInfo(_: Benchmark) !platform.OsInfo { - return try platform.getSystemInfo(); - } -}; +/// Get a copy of the system information, cpu type, cores, memory, etc. +pub fn getSystemInfo() !platform.OsInfo { + return try platform.getSystemInfo(); +} /// Carries the results of a benchmark. The benchmark name and the recorded /// durations are available, and some basic statistics are automatically @@ -277,91 +295,61 @@ pub const Result = struct { name: []const u8, timings_ns: []const u64, - // Statistics stored behind a pointer so Results can be cheap to pass by - // value. - statistics: *const Statistics, - - const Statistics = struct { - total_ns: u64, - mean_ns: u64, - stddev_ns: u64, - min_ns: u64, - max_ns: u64, - percentiles: Percentiles, - }; - - const Percentiles = struct { - p75_ns: u64, - p99_ns: u64, - p995_ns: u64, - }; - pub fn init( allocator: std.mem.Allocator, name: []const u8, - timings_ns: []u64, + readings: Runner.Readings, ) !Result { - const len = timings_ns.len; - std.sort.heap(u64, timings_ns, {}, std.sort.asc(u64)); - - // Calculate total and mean runtime - var total_ns: u64 = 0; - for (timings_ns) |ns| total_ns += ns; - const mean_ns: u64 = if (0 < len) total_ns / len else 0; - - // Calculate standard deviation - const stddev_ns: u64 = blk: { - var nvar: u64 = 0; - for (timings_ns) |ns| { - const sd = if (ns < mean_ns) mean_ns - ns else ns - mean_ns; - nvar += sd * sd; - } - break :blk if (1 < len) std.math.sqrt(nvar / (len - 1)) else 0; - }; - - const statistics: *Statistics = try allocator.create(Statistics); - statistics.* = Statistics{ - .total_ns = total_ns, - .mean_ns = mean_ns, - .stddev_ns = stddev_ns, - .min_ns = if (len == 0) 0 else timings_ns[0], - .max_ns = if (len == 0) 0 else timings_ns[len - 1], - .percentiles = Percentiles{ - .p75_ns = if (len == 0) 0 else timings_ns[len * 75 / 100], - .p99_ns = if (len == 0) 0 else timings_ns[len * 99 / 100], - .p995_ns = if (len == 0) 0 else timings_ns[len * 995 / 1000], - }, - }; - + std.sort.heap(u64, readings.timings_ns, {}, std.sort.asc(u64)); return Result{ .allocator = allocator, .name = name, - .statistics = statistics, - .timings_ns = timings_ns, + .timings_ns = readings.timings_ns, }; } pub fn deinit(self: Result) void { self.allocator.free(self.timings_ns); - self.allocator.destroy(self.statistics); } /// Formats and prints the benchmark result in a human readable format. /// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) /// colors: Whether to pretty-print with ANSI colors or not. pub fn prettyPrint(self: Result, writer: anytype, colors: bool) !void { - const s = self.statistics; - const p = s.percentiles; - try format.prettyPrintName(self.name, writer); + var buf: [128]u8 = undefined; + + const s = Statistics(u64).init(self.timings_ns); + // Benchmark name, number of iterations, and total time + try writer.print("{s:<22} ", .{self.name}); try setColor(colors, writer, Color.cyan); - try format.prettyPrintTotalOperations(self.timings_ns.len, writer); - try format.prettyPrintTotalTime(s.total_ns, writer); + try writer.print("{d:<8} {s:<15}", .{ + self.timings_ns.len, + std.fmt.fmtDuration(s.total), + }); + // Mean + standard deviation try setColor(colors, writer, Color.green); - try format.prettyPrintAvgStd(s.mean_ns, s.stddev_ns, writer); + try writer.print("{s:<23}", .{ + try std.fmt.bufPrint(&buf, "{:.3} ± {:.3}", .{ + std.fmt.fmtDuration(s.mean), + std.fmt.fmtDuration(s.stddev), + }), + }); + // Minimum and maximum try setColor(colors, writer, Color.blue); - try format.prettyPrintMinMax(s.min_ns, s.max_ns, writer); + try writer.print("{s:<29}", .{ + try std.fmt.bufPrint(&buf, "({:.3} ... {:.3})", .{ + std.fmt.fmtDuration(s.min), + std.fmt.fmtDuration(s.max), + }), + }); + // Percentiles try setColor(colors, writer, Color.cyan); - try format.prettyPrintPercentiles(p.p75_ns, p.p99_ns, p.p995_ns, writer); + try writer.print("{:<10} {:<10} {:<10}", .{ + std.fmt.fmtDuration(s.percentiles.p75), + std.fmt.fmtDuration(s.percentiles.p99), + std.fmt.fmtDuration(s.percentiles.p995), + }); + // End of line try setColor(colors, writer, Color.reset); try writer.writeAll("\n"); } @@ -371,105 +359,16 @@ pub const Result = struct { } pub fn writeJSON(self: Result, writer: anytype) !void { - const s = self.statistics; - const p = s.percentiles; - try std.fmt.format( - writer, + const timings_ns_stats = Statistics(u64).init(self.timings_ns); + try writer.print( \\{{ "name": "{s}", - \\ "units": "nanoseconds", - \\ "total": {d}, - \\ "mean": {d}, - \\ "stddev": {d}, - \\ "min": {d}, - \\ "max": {d}, - \\ "percentiles": {{"p75": {d}, "p99": {d}, "p995": {d} }}, - \\ "timings": [ + \\ "timing_statistics": {}, "timings": {} }} , .{ std.fmt.fmtSliceEscapeLower(self.name), - s.total_ns, - s.mean_ns, - s.stddev_ns, - s.min_ns, - s.max_ns, - p.p75_ns, - p.p99_ns, - p.p995_ns, + statistics.fmtJSON(u64, "nanoseconds", timings_ns_stats), + format.fmtJSONArray(u64, self.timings_ns), }, ); - for (self.timings_ns, 0..) |ns, i| { - if (0 < i) try writer.writeAll(", "); - try std.fmt.format(writer, "{d}", .{ns}); - } - try writer.writeAll("]}"); } }; - -test "Result" { - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 0), r.statistics.mean_ns); - try expectEq(@as(u64, 0), r.statistics.stddev_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - try timings_ns.append(1); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 1), r.statistics.mean_ns); - try expectEq(@as(u64, 0), r.statistics.stddev_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - try timings_ns.append(1); - for (1..16) |i| try timings_ns.append(i); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 7), r.statistics.mean_ns); - try expectEq(@as(u64, 4), r.statistics.stddev_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - try timings_ns.append(1); - for (1..101) |i| try timings_ns.append(i); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 50), r.statistics.mean_ns); - try expectEq(@as(u64, 29), r.statistics.stddev_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - for (0..10) |_| try timings_ns.append(1); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 1), r.statistics.mean_ns); - try expectEq(@as(u64, 0), r.statistics.stddev_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - for (0..100) |i| try timings_ns.append(i); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 75), r.statistics.percentiles.p75_ns); - try expectEq(@as(u64, 99), r.statistics.percentiles.p99_ns); - try expectEq(@as(u64, 99), r.statistics.percentiles.p995_ns); - } - - { - var timings_ns = std.ArrayList(u64).init(std.testing.allocator); - for (0..100) |i| try timings_ns.append(i); - std.mem.reverse(u64, timings_ns.items); - const r = try Result.init(std.testing.allocator, "r", try timings_ns.toOwnedSlice()); - defer r.deinit(); - try expectEq(@as(u64, 75), r.statistics.percentiles.p75_ns); - try expectEq(@as(u64, 99), r.statistics.percentiles.p99_ns); - try expectEq(@as(u64, 99), r.statistics.percentiles.p995_ns); - } -}