diff --git a/examples/basic.zig b/examples/basic.zig index a6784c7..9ffb123 100644 --- a/examples/basic.zig +++ b/examples/basic.zig @@ -25,4 +25,5 @@ test "bench test basic" { }; defer benchmarkResults.results.deinit(); try zbench.run(myBenchmark, &bench, &benchmarkResults); + try benchmarkResults.prettyPrint(); } diff --git a/examples/bubble_sort.zig b/examples/bubble_sort.zig index 54f77cc..4e801e0 100644 --- a/examples/bubble_sort.zig +++ b/examples/bubble_sort.zig @@ -28,4 +28,5 @@ test "bench test bubbleSort" { }; defer benchmarkResults.results.deinit(); try zbench.run(myBenchmark, &bench, &benchmarkResults); + try benchmarkResults.prettyPrint(); } diff --git a/examples/sleep.zig b/examples/sleep.zig index 7cfbf06..6a31a85 100644 --- a/examples/sleep.zig +++ b/examples/sleep.zig @@ -18,4 +18,5 @@ test "bench test sleepy" { }; defer benchmarkResults.results.deinit(); try zbench.run(sleepBenchmark, &bench, &benchmarkResults); + try benchmarkResults.prettyPrint(); } diff --git a/util/color.zig b/util/color.zig index cbd285d..f99e383 100644 --- a/util/color.zig +++ b/util/color.zig @@ -8,6 +8,7 @@ pub const Color = enum { magenta, cyan, reset, + none, // Return the ANSI escape code for this color. pub fn code(self: Color) []const u8 { @@ -19,11 +20,7 @@ pub const Color = enum { .magenta => "\x1b[35m", .cyan => "\x1b[36m", .reset => "\x1b[0m", + .none => "", }; } }; - -pub fn colorPrint(color: Color, text: []const u8) !void { - const stdout = std.io.getStdOut().writer(); - try stdout.print("{}{}{}", .{ color.code(), text, Color.reset.code() }); -} diff --git a/util/format.zig b/util/format.zig index 2d58a27..b4e636d 100644 --- a/util/format.zig +++ b/util/format.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const Color = @import("./color.zig").Color; pub fn duration(buffer: []u8, d: u64) ![]u8 { const units = [_][]const u8{ "ns", "µs", "ms", "s" }; @@ -18,3 +19,66 @@ pub fn duration(buffer: []u8, d: u64) ![]u8 { return formatted; } + +/// Pretty-prints the name of the benchmark +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintName(name: []const u8, writer: anytype, color: Color) !void { + try writer.print("{s}{s:<22}{s} ", .{ color.code(), name, Color.reset.code() }); +} + +/// Pretty-prints the number of total operations (or runs) of the benchmark performed +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintTotalOperations(total_operations: u64, writer: anytype, color: Color) !void { + try writer.print("{s}{d:<8}{s} ", .{ color.code(), total_operations, Color.reset.code() }); +} + +/// Pretty-prints the total time it took to perform all the runs +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintTotalTime(total_time: u64, writer: anytype, color: Color) !void { + var buffer: [128]u8 = undefined; + const str = try duration(buffer[0..], total_time); + + try writer.print("{s}{s:<14}{s} ", .{ color.code(), str, Color.reset.code() }); +} + +/// Pretty-prints the average (arithmetic mean) and the standard deviation of the durations +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintAvgStd(avg: u64, stdd: u64, writer: anytype, color: Color) !void { + var buffer: [128]u8 = undefined; + var avg_stdd_offset = (try duration(buffer[0..], avg)).len; + avg_stdd_offset += (try std.fmt.bufPrint(buffer[avg_stdd_offset..], " ± ", .{})).len; + avg_stdd_offset += (try duration(buffer[avg_stdd_offset..], stdd)).len; + const str = buffer[0..avg_stdd_offset]; + + try writer.print("{s}{s:<22}{s} ", .{ color.code(), str, Color.reset.code() }); +} + +/// Pretty-prints the minumim and maximum duration +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintMinMax(min: u64, max: u64, writer: anytype, color: Color) !void { + var min_buffer: [128]u8 = undefined; + const min_str = try duration(min_buffer[0..], min); + + var max_buffer: [128]u8 = undefined; + const max_str = try duration(max_buffer[0..], max); + + var buffer: [128]u8 = undefined; + const str = try std.fmt.bufPrint(buffer[0..], "({s} ... {s})", .{ min_str, max_str }); + + try writer.print("{s}{s:<28}{s} ", .{ color.code(), str, Color.reset.code() }); +} + +/// Pretty-prints the 75th, 99th and 99.5th percentile of the durations +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintPercentiles(p75: u64, p99: u64, p995: u64, writer: anytype, color: Color) !void { + var p75_buffer: [128]u8 = undefined; + const p75_str = try duration(p75_buffer[0..], p75); + + var p99_buffer: [128]u8 = undefined; + const p99_str = try duration(p99_buffer[0..], p99); + + var p995_buffer: [128]u8 = undefined; + const p995_str = try duration(p995_buffer[0..], p995); + + try writer.print("{s}{s:<10} {s:<10} {s:<10}{s} ", .{ color.code(), p75_str, p99_str, p995_str, Color.reset.code() }); +} diff --git a/zbench.zig b/zbench.zig index 712639f..91e21a9 100644 --- a/zbench.zig +++ b/zbench.zig @@ -11,6 +11,14 @@ const format = @import("./util/format.zig"); /// Benchmark is a type representing a single benchmark session. /// It provides metrics and utilities for performance measurement. pub const Benchmark = struct { + /// Used to represent the 75th, 99th and 99.5th percentiles of the recorded durations, + /// generated by `Benchmark.calculatePercentiles`. + pub const Percentiles = struct { + p75: u64, + p99: u64, + p995: u64, + }; + /// Name of the benchmark. name: []const u8, /// Number of iterations to be performed in the benchmark. @@ -83,12 +91,6 @@ pub const Benchmark = struct { self.total_operations = ops; } - pub const Percentiles = struct { - p75: u64, - p99: u64, - p995: u64, - }; - pub fn quickSort(items: []u64, low: usize, high: usize) void { if (low < high) { const pivotIndex = partition(items, low, high); @@ -114,7 +116,9 @@ pub const Benchmark = struct { return i; } - /// Calculate the p75, p99, and p995 durations + /// Calculate the 75th, 99th and 99.5th percentiles of the durations. They represent the timings below + /// which 75%, 99% and 99.5% of the other measurments would lie (respectively) when timings are + /// sorted in increasing order. pub fn calculatePercentiles(self: Benchmark) Percentiles { // quickSort might fail with an empty input slice, so safety checks first const len = self.durations.items.len; @@ -139,9 +143,13 @@ pub const Benchmark = struct { } /// Prints a report of total operations and timing statistics. + /// (Similar to BenchmarkResult.prettyPrint) pub fn report(self: Benchmark) !void { const percentiles = self.calculatePercentiles(); + var total_time_buffer: [128]u8 = undefined; + const total_time_str = try format.duration(total_time_buffer[0..], self.elapsed()); + var p75_buffer: [128]u8 = undefined; const p75_str = try format.duration(p75_buffer[0..], percentiles.p75); @@ -167,19 +175,16 @@ pub const Benchmark = struct { const min_max_str = try std.fmt.bufPrint(min_max_buffer[0..], "({s} ... {s})", .{ min_str, max_str }); const stdout = std.io.getStdOut().writer(); - try stdout.print( - "\n{s:<22} {s:<8} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10}\n", - .{ "benchmark", "runs", "time (avg ± σ)", "(min ... max)", "p75", "p99", "p995" }, - ); + prettyPrintHeader(); try stdout.print("---------------------------------------------------------------------------------------------------------------\n", .{}); try stdout.print( - "{s:<22} \x1b[90m{d:<8} \x1b[33m{s:<22} \x1b[95m{s:<28} \x1b[90m{s:<10} {s:<10} {s:<10}\x1b[0m\n", - .{ self.name, self.total_operations, avg_std_str, min_max_str, p75_str, p99_str, p995_str }, + "{s:<22} \x1b[90m{d:<8} \x1b[90m{s:<10} \x1b[33m{s:<22} \x1b[95m{s:<28} \x1b[90m{s:<10} {s:<10} {s:<10}\x1b[0m\n\n", + .{ self.name, self.total_operations, total_time_str, avg_std_str, min_max_str, p75_str, p99_str, p995_str }, ); try stdout.print("\n", .{}); } - /// Calculate the average duration + /// Calculate the average (more precisely arithmetic mean) of the durations pub fn calculateAverage(self: Benchmark) u64 { // prevent division by zero const len = self.durations.items.len; @@ -195,7 +200,8 @@ pub const Benchmark = struct { return avg; } - /// Calculate the standard deviation of the durations + /// Calculate the standard deviation of the durations. An estimate for the average *deviation* + /// from the average duration. pub fn calculateStd(self: Benchmark) u64 { if (self.durations.items.len <= 1) return 0; @@ -210,7 +216,7 @@ pub const Benchmark = struct { nvar += @bitCast((d - a) * (d - a)); } - // We are using the non-biased estimator for the variance; sum(X - μ)^2 / (n - 1) + // We are using the non-biased estimator for the variance; sum(Xi - μ)^2 / (n - 1) return std.math.sqrt(nvar / (self.durations.items.len - 1)); } }; @@ -219,46 +225,99 @@ pub const Benchmark = struct { /// It takes a pointer to a Benchmark object. pub const BenchFunc = fn (*Benchmark) void; -/// BenchmarkResult stores the result of a single benchmark. -/// It includes the name and the total duration of the benchmark. +/// BenchmarkResult stores the resulting computed metrics/statistics from a benchmark pub const BenchmarkResult = struct { - /// Name of the benchmark. + const Self = @This(); + const Color = c.Color; + + /// Name of the benchmark name: []const u8, - /// Total duration of the benchmark in nanoseconds. - duration: u64, + /// 75th, 99th and 99.5th percentiles of the recorded durations. They represent the timings below + /// which 75%, 99% and 99.5% of the other measurments would lie, respectively, when timings + /// are sorted in increasing order. + percentiles: Benchmark.Percentiles, + /// The average (more precisely arithmetic mean) of the recorded durations + avg_duration: usize, + /// The standard-deviation of the recorded durations (an estimate for the average *deviation* from + /// the average duration). + std_duration: usize, + /// The minimum among the recorded durations + min_duration: usize, + /// The maximum among the recorded durations + max_duration: usize, + /// The total amount of operations (or runs) performed of the benchmark + total_operations: usize, + /// Total time for all the operations (or runs) of the benchmark combined + total_time: usize, + + /// Formats and prints the benchmark-result in a readable format. + /// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) + /// header: Whether to pretty-print the header or not + pub fn prettyPrint(self: Self, writer: anytype, header: bool) !void { + if (header) try prettyPrintHeader(writer); + + try format.prettyPrintName(self.name, writer, Color.none); + try format.prettyPrintTotalOperations(self.total_operations, writer, Color.cyan); + try format.prettyPrintTotalTime(self.total_time, writer, Color.cyan); + try format.prettyPrintAvgStd(self.avg_durations, self.std_durations, writer, Color.green); + try format.prettyPrintMinMax(self.min_durations, self.max_durations, writer, Color.blue); + try format.prettyPrintPercentiles(self.percentiles.p75, self.percentiles.p99, self.percentiles.p995, writer, Color.cyan); + + _ = try writer.write("\n"); + } }; +/// Pretty-prints the header for the result pretty-print table +/// writer: Type that has the associated method print (for example std.io.getStdOut.writer()) +pub fn prettyPrintHeader(writer: anytype) !void { + try writer.print( + "\n{s:<22} {s:<8} {s:<14} {s:<22} {s:<28} {s:<10} {s:<10} {s:<10}\n", + .{ "benchmark", "runs", "total time", "time/run (avg ± σ)", "(min ... max)", "p75", "p99", "p995" }, + ); + try writer.print("-----------------------------------------------------------------------------------------------------------------------------\n", .{}); +} + /// BenchmarkResults acts as a container for multiple benchmark results. /// It provides functionality to format and print these results. pub const BenchmarkResults = struct { + const Color = c.Color; + /// A dynamic list of BenchmarkResult objects. results: std.ArrayList(BenchmarkResult), + /// A handle to a buffered stdout-writer. Used for printing-operations + out_stream: std.io.BufferedWriter(1024, @TypeOf(std.io.getStdOut().writer())) = .{ .unbuffered_writer = std.io.getStdOut().writer() }, - /// Determines the color representation based on the duration of the benchmark. - /// duration: The duration to evaluate. - pub fn getColor(self: *const BenchmarkResults, duration: u64) c.Color { - const max_duration = @max(self.results.items[0].duration, self.results.items[self.results.items.len - 1].duration); - const min_duration = @min(self.results.items[0].duration, self.results.items[self.results.items.len - 1].duration); + /// Determines the color representation based on the total-time of the benchmark. + /// total_time: The total-time to evaluate. + pub fn getColor(self: *const BenchmarkResults, total_time: u64) Color { + const max_total_time = @max(self.results.items[0].total_time, self.results.items[self.results.items.len - 1].total_time); + const min_total_time = @min(self.results.items[0].total_time, self.results.items[self.results.items.len - 1].total_time); - if (duration <= min_duration) return c.Color.green; - if (duration >= max_duration) return c.Color.red; + if (total_time <= min_total_time) return Color.green; + if (total_time >= max_total_time) return Color.red; - const prop = (duration - min_duration) * 100 / (max_duration - min_duration + 1); + const prop = (total_time - min_total_time) * 100 / (max_total_time - min_total_time + 1); - if (prop < 50) return c.Color.green; - if (prop < 75) return c.Color.yellow; + if (prop < 50) return Color.green; + if (prop < 75) return Color.yellow; - return c.Color.red; + return Color.red; } /// Formats and prints the benchmark results in a readable format. - pub fn prettyPrint(self: BenchmarkResults) !void { - const stdout = std.io.getStdOut().writer(); - stdout.print("--------------------------------------------------------------------------------------\n", .{}); - + pub fn prettyPrint(self: *BenchmarkResults) !void { + var writer = self.out_stream.writer(); + try prettyPrintHeader(writer); for (self.results.items) |result| { - try stdout.print("{s}", .{result.name}); + try format.prettyPrintName(result.name, writer, Color.none); + try format.prettyPrintTotalOperations(result.total_operations, writer, Color.cyan); + try format.prettyPrintTotalTime(result.total_time, writer, Color.cyan); + try format.prettyPrintAvgStd(result.avg_duration, result.std_duration, writer, Color.green); + try format.prettyPrintMinMax(result.min_duration, result.max_duration, writer, Color.blue); + try format.prettyPrintPercentiles(result.percentiles.p75, result.percentiles.p99, result.percentiles.p995, writer, Color.cyan); } + + try self.out_stream.flush(); } }; @@ -316,13 +375,17 @@ pub fn run(comptime func: BenchFunc, bench: *Benchmark, benchResult: *BenchmarkR bench.stop(); } + bench.setTotalOperations(bench.N); + const elapsed = bench.elapsed(); try benchResult.results.append(BenchmarkResult{ .name = bench.name, - .duration = elapsed, + .percentiles = bench.calculatePercentiles(), + .avg_duration = bench.calculateAverage(), + .std_duration = bench.calculateStd(), + .min_duration = bench.min_duration, + .max_duration = bench.max_duration, + .total_time = elapsed, + .total_operations = bench.total_operations, }); - - bench.setTotalOperations(bench.N); - - try bench.report(); }