From fb0373b89a1a6df1cbb6149920c06c09554b5090 Mon Sep 17 00:00:00 2001 From: filzrev <103790468+filzrev@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:47:13 +0900 Subject: [PATCH] feat: add JobOrderPolicy option to sort jobs numeric order --- .../Attributes/OrdererAttribute.cs | 5 +- src/BenchmarkDotNet/Jobs/JobComparer.cs | 93 ++++++++++- src/BenchmarkDotNet/Order/DefaultOrderer.cs | 8 +- src/BenchmarkDotNet/Order/JobOrderPolicy.cs | 14 ++ .../Order/JobOrderTests.cs | 146 ++++++++++++++++++ 5 files changed, 258 insertions(+), 8 deletions(-) create mode 100644 src/BenchmarkDotNet/Order/JobOrderPolicy.cs create mode 100644 tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs diff --git a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs index 83b5051895..b20a9108ca 100644 --- a/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/OrdererAttribute.cs @@ -9,9 +9,10 @@ public class OrdererAttribute : Attribute, IConfigSource { public OrdererAttribute( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { - Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy)); + Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy)); } public IConfig Config { get; } diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 0a0c287523..ac1539ce7c 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -1,12 +1,23 @@ -using System; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Order; +using System; using System.Collections.Generic; -using BenchmarkDotNet.Characteristics; namespace BenchmarkDotNet.Jobs { internal class JobComparer : IComparer, IEqualityComparer { - public static readonly JobComparer Instance = new JobComparer(); + private readonly IComparer Comparer; + + public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default); + public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric); + + public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) + { + Comparer = jobOrderPolicy == JobOrderPolicy.Default + ? StringComparer.Ordinal + : new NumericStringComparer(); // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater. + } public int Compare(Job x, Job y) { @@ -39,7 +50,7 @@ public int Compare(Job x, Job y) continue; } - int compare = string.CompareOrdinal( + int compare = Comparer.Compare( presenter.ToPresentation(x, characteristic), presenter.ToPresentation(y, characteristic)); if (compare != 0) @@ -52,5 +63,79 @@ public int Compare(Job x, Job y) public bool Equals(Job x, Job y) => Compare(x, y) == 0; public int GetHashCode(Job obj) => obj.Id.GetHashCode(); + + internal class NumericStringComparer : IComparer + { + public int Compare(string? x, string? y) + { + if (ReferenceEquals(x, y)) return 0; + if (x == null) return -1; + if (y == null) return 1; + + ReadOnlySpan spanX = x.AsSpan(); + ReadOnlySpan spanY = y.AsSpan(); + + int i = 0, j = 0; + + while (i < spanX.Length && j < spanY.Length) + { + char cx = spanX[i]; + char cy = spanY[j]; + + if (!char.IsDigit(cx) || !char.IsDigit(cy)) + { + int cmp = cx.CompareTo(cy); + if (cmp != 0) + return cmp; + + i++; + j++; + continue; + } + + int ixStart = i; + int iyStart = j; + + // Skip leading zeros + while (ixStart < spanX.Length && spanX[ixStart] == '0') ixStart++; + while (iyStart < spanY.Length && spanY[iyStart] == '0') iyStart++; + + int ix = ixStart; + int iy = iyStart; + + // Skip digits + while (ix < spanX.Length && char.IsDigit(spanX[ix])) ix++; + while (iy < spanY.Length && char.IsDigit(spanY[iy])) iy++; + + int lenX = ix - ixStart; + int lenY = iy - iyStart; + + // Compare by digits length + if (lenX != lenY) + return lenX.CompareTo(lenY); + + // Compare digits + for (int k = 0; k < lenX; k++) + { + int cmp = spanX[ixStart + k].CompareTo(spanY[iyStart + k]); + if (cmp != 0) + return cmp; + } + + // Compare by leading zeros + int leadingZerosX = ixStart - i; + int leadingZerosY = iyStart - j; + if (leadingZerosX != leadingZerosY) + return 0; // Leading zero differences are ignored (`CompareOptions.NumericOrdering` behavior of .NET) + + // Move to the next character after the digits + i = ix; + j = iy; + } + + // Compare remaining chars + return (spanX.Length - i).CompareTo(spanY.Length - j); + } + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Order/DefaultOrderer.cs b/src/BenchmarkDotNet/Order/DefaultOrderer.cs index 656de7ce98..d0dc2d475b 100644 --- a/src/BenchmarkDotNet/Order/DefaultOrderer.cs +++ b/src/BenchmarkDotNet/Order/DefaultOrderer.cs @@ -19,7 +19,7 @@ public class DefaultOrderer : IOrderer private readonly IComparer categoryComparer = CategoryComparer.Instance; private readonly IComparer paramsComparer = ParameterComparer.Instance; - private readonly IComparer jobComparer = JobComparer.Instance; + private readonly IComparer jobComparer; private readonly IComparer targetComparer; public SummaryOrderPolicy SummaryOrderPolicy { get; } @@ -27,10 +27,14 @@ public class DefaultOrderer : IOrderer public DefaultOrderer( SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default, - MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared) + MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared, + JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default) { SummaryOrderPolicy = summaryOrderPolicy; MethodOrderPolicy = methodOrderPolicy; + jobComparer = jobOrderPolicy == JobOrderPolicy.Default + ? JobComparer.Instance + : JobComparer.Numeric; targetComparer = new DescriptorComparer(methodOrderPolicy); } diff --git a/src/BenchmarkDotNet/Order/JobOrderPolicy.cs b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs new file mode 100644 index 0000000000..be1d66186d --- /dev/null +++ b/src/BenchmarkDotNet/Order/JobOrderPolicy.cs @@ -0,0 +1,14 @@ +namespace BenchmarkDotNet.Order; + +public enum JobOrderPolicy +{ + /// + /// Compare job characteristics in ordinal order. + /// + Default, + + /// + /// Compare job characteristics in numeric order. + /// + Numeric, +} diff --git a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs new file mode 100644 index 0000000000..c0e645ea22 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs @@ -0,0 +1,146 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.CsProj; +using System.Linq; +using Xunit; + +namespace BenchmarkDotNet.Tests.Order; + +public class JobOrderTests +{ + [Fact] + public void TestJobOrders_ByJobId() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core80) + .WithId("v1.4.1"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90) + .WithId("v1.4.10"), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core10_0) + .WithId("v1.4.2"), + ]; + + // Verify jobs are sorted by JobId's ordinal order. + { + // Act + var comparer = JobComparer.Instance; + var results = jobs.OrderBy(x => x, comparer) + .Select(x => x.Job.Id) + .ToArray(); + + // Assert + Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results); + } + + // Verify jobs are sorted by JobId's numeric order. + { + // Act + var comparer = JobComparer.Numeric; + var results = jobs.OrderBy(d => d, comparer) + .Select(x => x.Job.Id) + .ToArray(); + // Assert + Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results); + } + } + + [Fact] + public void TestJobOrders_ByRuntime() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) + .WithRuntime(CoreRuntime.Core80), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) + .WithRuntime(CoreRuntime.Core90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) + .WithRuntime(CoreRuntime.Core10_0), + ]; + + // Act + // Verify jobs are sorted by Runtime's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.Environment.GetRuntime().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CoreRuntime.Core80.Name, + CoreRuntime.Core90.Name, + CoreRuntime.Core10_0.Name + }; + Assert.Equal(expected, results); + } + + [Fact] + public void TestJobOrders_ByToolchain() + { + // Arrange + Job[] jobs = + [ + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90), + Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80), + ]; + + // Act + // Verify jobs are sorted by Toolchain's numeric order. + var results = jobs.OrderBy(d => d, JobComparer.Numeric) + .Select(x => x.Job.GetToolchain().Name) + .ToArray(); + + // Assert + var expected = new[] + { + CsProjCoreToolchain.NetCoreApp80.Name, + CsProjCoreToolchain.NetCoreApp90.Name, + CsProjCoreToolchain.NetCoreApp10_0.Name, + }; + Assert.Equal(expected, results); + } + + [Theory] + [InlineData("item1", "item1", 0)] + [InlineData("item123", "item123", 0)] + // Compare different values + [InlineData("item1", "item2", -1)] + [InlineData("item2", "item1", 1)] + [InlineData("item2", "item10", -1)] + [InlineData("item10", "item2", 1)] + [InlineData("item1a", "item1b", -1)] + [InlineData("item1b", "item1a", 1)] + [InlineData("item", "item1", -1)] + [InlineData("item10", "item", 1)] + [InlineData(".NET 8", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 8", 1)] + [InlineData("v1.4.1", "v1.4.10", -1)] + [InlineData("v1.4.10", "v1.4.2", 1)] + // Compare zero paddeed numeric string. + [InlineData("item01", "item1", 0)] + [InlineData("item001", "item1", 0)] + [InlineData("item1", "item001", 0)] + [InlineData("item1", "item01", 0)] + [InlineData("item9", "item09", 0)] + [InlineData(".NET 08", ".NET 10", -1)] + [InlineData(".NET 10", ".NET 08", 1)] + // Arguments that contains null + [InlineData(null, "a", -1)] + [InlineData("a", null, 1)] + [InlineData(null, null, 0)] + public void TestNumericComparer(string? a, string? b, int expectedSign) + { + int result = new JobComparer.NumericStringComparer().Compare(a, b); + Assert.Equal(expectedSign, NormalizeSign(result)); + + static int NormalizeSign(int value) + => value == 0 ? 0 : value < 0 ? -1 : 1; + } +}