diff --git a/ci/perf/gpuc1_bench.csv b/ci/perf/gpuc1_bench.csv index 72c6eff61..7e6988956 100644 --- a/ci/perf/gpuc1_bench.csv +++ b/ci/perf/gpuc1_bench.csv @@ -1,13 +1,13 @@ test case,benchmark name,samples,iterations,estimated,mean,low mean,high mean,std dev,low std dev,high std dev,raw -"benchmark intrusive graph dependency handling, N=1",creating nodes,100,2863,2290400,3.3819,3.3815,3.3828,0.0029,0.0017,0.0057,"3.3797,3.3800,3.4045,3.3832,3.3832,3.3835,3.3797,3.3797,3.3832,3.3832,3.3835,3.3797,3.3797,3.3797,3.3832,3.3835,3.3832,3.3797,3.3797,3.3832,3.3835,3.3832,3.3797,3.3800,3.3832,3.3835,3.3832,3.3797,3.3800,3.3832,3.3835,3.3832,3.3797,3.3800,3.3832,3.3835,3.3832,3.3797,3.3800,3.3797,3.3835,3.3832,3.3832,3.3800,3.3797,3.3835,3.3832,3.3832,3.3800,3.3797,3.3832,3.3832,3.3832,3.3800,3.3797,3.3832,3.3832,3.3832,3.3800,3.3797,3.3832,3.3832,3.3832,3.3800,3.3797,3.3797,3.3835,3.3832,3.3835,3.3797,3.3797,3.3835,3.3832,3.3800,3.3797,3.3797,3.3835,3.3832,3.3835,3.3797,3.3797,3.3835,3.3832,3.3835,3.3797,3.3797,3.3835,3.3832,3.3835,3.3797,3.3797,3.3800,3.3832,3.3835,3.3797,3.3797,3.3800,3.3832,3.3832,3.3797" -"benchmark intrusive graph dependency handling, N=1",creating and adding dependencies,100,852,2470800,23.8438,23.8392,23.8559,0.0359,0.0106,0.0780,"23.8462,23.8451,24.1631,23.9038,23.8345,23.8333,23.8462,23.8451,23.8462,23.8451,23.8462,23.8451,23.8345,23.7042,23.8451,23.8462,23.8451,23.8462,23.8451,23.8345,23.8333,23.8345,23.8333,23.8462,23.8451,23.8462,23.8451,23.8462,23.8333,23.8333,23.8333,23.8333,23.8451,23.8451,23.8462,23.8451,23.8462,23.8451,23.8345,23.8333,23.8345,23.8333,23.8462,23.8451,23.8462,23.8451,23.8462,23.8333,23.8345,23.8333,23.8345,23.8451,23.8462,23.8451,23.8462,23.8451,23.8333,23.8333,23.8333,23.8333,23.8451,23.8462,23.8451,23.8462,23.8451,23.8462,23.8333,23.8345,23.8333,23.8462,23.8451,23.8462,23.8451,23.8462,23.8451,23.8345,23.8333,23.8345,23.8451,23.8462,23.8451,23.8462,23.8451,23.8451,23.8333,23.8333,23.8333,23.8451,23.8462,23.8451,23.8462,23.8451,23.8462,23.8333,23.8345,23.8333,23.8462,23.8451,23.8462,23.8451" -"benchmark intrusive graph dependency handling, N=1",adding and removing dependencies,100,1326,2386800,20.5839,20.5408,20.7911,0.4157,0.0193,0.9910,"20.5422,20.5271,20.7089,20.5498,20.5196,20.5505,20.5498,20.5279,20.5422,20.5505,20.5271,20.5505,20.5505,20.5347,20.5505,20.5422,20.5505,20.5271,20.5430,20.5430,20.5271,20.5505,20.5422,20.5204,20.5422,20.5430,20.5430,20.5271,20.5505,20.5498,20.5279,20.5422,20.5430,20.5505,20.5196,20.5505,20.5498,24.7134,20.5505,20.6787,20.5271,20.5430,20.5279,20.5498,20.5279,20.5347,20.5279,20.5422,20.5505,20.5271,20.5498,20.5279,20.5422,20.5505,20.5347,20.5505,20.5422,20.5271,20.5430,20.5271,20.5430,20.5422,20.5279,20.5422,20.5279,20.5430,20.5422,20.5279,20.5422,20.5279,20.5422,20.5430,20.5279,20.5498,20.5505,20.5271,20.5430,20.5271,20.5430,20.5430,20.5271,20.5430,20.5498,20.5279,20.5422,20.5279,20.5498,20.5422,20.5279,20.5422,20.5505,20.5271,20.5430,20.5271,20.5422,20.5430,20.5271,20.5430,20.5498,20.5279" -"benchmark intrusive graph dependency handling, N=1",checking for dependencies,100,12427,2485400,2.9010,2.8966,2.9232,0.0441,0.0001,0.0966,"2.8966,2.8966,2.8990,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8967,2.8958,2.8966,2.8966,3.3400,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8958,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8967,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8967,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8967,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8967,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966,2.8966" -"benchmark intrusive graph dependency handling, N=10",creating nodes,100,587,2465400,43.6176,43.5770,43.6601,0.2116,0.1884,0.2470,"43.4174,43.5196,44.2896,43.7581,43.9813,43.5383,43.9114,43.5213,43.9983,43.5366,44.0835,43.4855,43.8279,43.9302,43.4855,43.6405,43.5707,43.9302,43.7257,43.7751,43.2811,43.6729,43.2300,43.4514,43.5043,43.5366,43.6917,43.4174,43.7768,43.3833,43.9131,43.7240,43.8279,43.3152,43.8790,43.2811,43.9131,43.2470,43.7257,43.4685,43.6917,43.9114,43.7087,43.5366,43.6235,43.5707,43.5894,43.6048,43.4003,43.5554,43.5026,43.6576,43.4855,43.7087,43.4344,43.9302,43.7240,43.7939,43.2981,43.6576,43.3322,43.8620,43.2641,43.6899,43.7257,43.7257,43.9114,43.4872,43.4514,43.6746,43.5196,43.6218,43.5707,43.5877,43.5894,43.4003,43.5366,43.5196,43.6729,43.3509,43.5877,43.4514,43.9131,43.6235,43.6048,43.3152,43.6405,43.3322,43.8620,43.2981,43.9131,43.7240,43.9131,43.2470,43.7087,43.4685,43.6746,43.5196,43.4344,43.4872" -"benchmark intrusive graph dependency handling, N=10",creating and adding dependencies,100,95,2498500,246.1885,245.8262,247.4747,3.1056,0.8737,7.1184,"246.1263,246.1263,250.8737,244.0211,243.8105,243.7053,245.7053,245.8105,246.1263,246.2421,246.1263,246.1263,246.1263,246.5474,246.2316,246.1263,246.1368,246.1263,246.1263,246.1368,246.1263,275.5474,246.7579,246.2316,246.1263,246.1263,246.1263,246.2316,246.1368,246.1263,246.1263,246.1263,246.1263,246.2316,246.1263,246.2421,246.2316,245.7053,246.2316,246.1368,246.2316,246.1263,246.1368,246.1263,246.2316,245.8105,245.8211,246.1263,246.2316,246.1263,246.1263,246.1263,246.2316,245.8211,245.8105,246.1263,246.2316,246.1368,246.1263,246.1263,246.2316,245.8105,245.8105,246.1263,246.2421,246.1263,246.1263,246.1263,246.2421,243.2737,242.6421,243.6000,243.6000,243.4947,243.5895,243.6000,243.6000,246.1263,246.1368,246.1263,246.2316,246.1263,246.2316,246.1263,246.1263,246.1368,246.2316,246.1263,246.1368,246.1263,246.1263,246.1263,246.2421,246.1263,246.1263,246.2421,246.1263,246.1263,246.2316,246.1368" -"benchmark intrusive graph dependency handling, N=10",adding and removing dependencies,100,108,2494800,226.1163,225.5790,228.6562,5.1209,0.3546,12.1870,"225.3148,225.3148,228.0000,226.0556,226.0556,226.0556,226.0556,226.0648,226.0556,226.0556,226.0556,226.0648,226.0556,226.0556,225.9630,226.0556,226.0556,226.0556,226.0556,225.9630,226.0648,226.0556,226.0556,226.0556,226.0556,226.0556,226.0556,226.0556,226.0556,226.0648,226.0556,226.0556,226.0556,226.0648,226.0556,226.0556,226.0556,276.8981,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.4074,225.3148,225.4074,225.3148,225.3148,225.3148,225.4074,225.3148,225.3148,225.3148,225.3148,225.3148,225.4074,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.4074,225.3148,225.3148,225.3148,225.4074,225.3148,225.4074,225.3148,225.3148,225.3148,225.4074,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.3148,225.4074,225.3148" -"benchmark intrusive graph dependency handling, N=10",checking for dependencies,100,732,2488800,35.9287,35.8127,36.1146,0.7330,0.5337,1.2984,"35.4877,35.2555,36.8975,36.4727,36.5560,36.4590,36.5546,36.4604,36.5683,36.4604,36.5560,36.4590,36.5560,36.4590,36.5683,36.4604,36.5546,36.4604,36.5560,36.4590,36.5697,36.4590,36.5560,36.4604,36.5683,36.4604,36.5683,36.4590,36.5560,36.4590,36.5560,36.4590,36.5546,36.4604,36.5956,36.4740,36.5697,36.4590,36.5560,36.4590,36.5697,40.8538,36.4727,35.1462,35.0232,35.0082,35.5833,35.5027,35.1448,35.5027,35.6380,35.5150,35.2008,35.4877,35.2281,35.5014,35.2828,35.5014,35.2691,35.4877,35.5984,35.5014,35.5970,35.5027,35.3374,35.5014,35.3511,35.5014,35.3784,35.4877,35.4057,35.5014,35.4331,35.5014,35.4467,35.5014,35.4740,35.5027,35.5970,35.4877,35.5970,35.5014,35.6243,35.5150,35.5984,35.5014,35.6107,35.5014,35.5984,35.5014,35.4877,35.1735,35.5014,35.5970,35.5014,35.1721,35.5014,35.2418,35.5014,35.5970" -"benchmark intrusive graph dependency handling, N=100",creating nodes,100,60,2490000,423.9813,418.8192,429.1558,26.4267,24.2945,31.5456,"391.0500,391.0333,455.1667,442.9667,442.4667,439.1333,441.3000,441.1333,442.6333,440.3000,442.1333,441.3167,441.6333,440.9667,442.8000,442.4833,439.1167,440.8000,439.1167,441.1333,440.1333,442.4667,439.6333,442.4667,441.3000,441.3000,442.3167,439.1333,440.8000,439.1167,441.9667,441.3000,441.6333,440.9667,442.6333,440.9833,439.4500,442.3000,439.1333,441.4667,441.3000,441.3000,440.6333,442.9667,441.3000,441.3000,442.6333,439.4667,442.9667,441.3000,442.9833,441.3000,441.6333,439.1333,440.9667,440.9667,442.6333,442.6333,439.1333,440.8000,441.3000,442.6333,441.3000,482.3833,395.5500,391.3833,391.3667,391.2000,391.2000,511.7667,391.0500,391.0333,391.0333,391.0500,391.0333,391.0500,391.0333,391.2167,391.0333,391.0333,391.0333,391.0333,391.0500,391.0333,391.0500,391.0333,391.0333,391.0333,391.0333,391.0500,391.0333,391.0500,391.0333,391.0500,391.0333,391.0333,391.2167,391.0333,391.0500,391.0333" -"benchmark intrusive graph dependency handling, N=100",creating and adding dependencies,100,6,2793600,4810.1767,4804.5383,4820.3183,37.5406,21.3230,61.9397,"4802.1667,4775.3333,5055.8333,4815.5000,4793.6667,4788.8333,4797.0000,4793.6667,4780.3333,4807.0000,4787.1667,4795.3333,4795.5000,4792.0000,4797.1667,4802.0000,4788.8333,4802.0000,4765.3333,4805.5000,4780.3333,4813.8333,4822.1667,4817.0000,4832.1667,4820.5000,4810.5000,4820.3333,4815.3333,4797.1667,4825.3333,4790.3333,4828.8333,4827.1667,4807.0000,4845.5000,4828.8333,4832.1667,4840.5000,4813.8333,4835.5000,4818.6667,4835.5000,4832.1667,4813.8333,4808.6667,4795.3333,4817.1667,4818.6667,4828.8333,4818.8333,4838.8333,4807.1667,4832.0000,4822.0000,4818.8333,4838.8333,4832.1667,4827.1667,4842.1667,4828.8333,4835.3333,4838.6667,4812.0000,4832.1667,4820.5000,4833.8333,4833.8333,4997.3333,4820.5000,4775.3333,4775.5000,4777.0000,4762.0000,4783.6667,4772.1667,4788.6667,4772.0000,4763.6667,4793.8333,4785.3333,4778.6667,4797.1667,4795.3333,4788.8333,4788.6667,4775.3333,4798.8333,4788.6667,4782.1667,4792.0000,4790.3333,4795.3333,4775.3333,4803.8333,4797.0000,4797.1667,4792.0000,4783.8333,4803.6667" -"benchmark intrusive graph dependency handling, N=100",adding and removing dependencies,100,6,2793600,4767.8167,4759.5333,4786.6967,60.8263,33.2226,122.7611,"4731.8333,4763.6667,4932.3333,4825.5000,4820.5000,4822.1667,4777.0000,4765.3333,4788.8333,4835.5000,4765.3333,4788.6667,4795.5000,4810.3333,4827.1667,4765.3333,4733.6667,4788.6667,4805.5000,4823.8333,4802.0000,4817.1667,4775.3333,4748.6667,4800.5000,4807.0000,4823.8333,4753.6667,4737.0000,4787.0000,4757.0000,4752.0000,4775.5000,4790.3333,4803.8333,4763.6667,4807.0000,4820.5000,4783.6667,4778.8333,4735.1667,4748.6667,4813.8333,4813.8333,5257.8333,4710.1667,4745.3333,4720.1667,4750.1667,4731.8333,4740.3333,4735.3333,4735.3333,4715.1667,4735.3333,4740.3333,4768.6667,4755.3333,4762.0000,4740.3333,4758.6667,4727.0000,4768.6667,4733.6667,4757.0000,4752.0000,4745.3333,4741.8333,4720.3333,4730.3333,4711.8333,4738.6667,4733.6667,4742.0000,4715.1667,4733.6667,4718.6667,4730.1667,4733.5000,4747.0000,4747.0000,4732.0000,4757.0000,4732.0000,4748.5000,4725.1667,4752.0000,4740.3333,4748.6667,4742.0000,4770.3333,4747.0000,4765.3333,4738.6667,4747.0000,4752.0000,4732.0000,4750.3333,4750.1667,4758.6667" -"benchmark intrusive graph dependency handling, N=100",checking for dependencies,100,13,2633800,2031.1808,2030.1715,2032.3292,5.4984,4.7051,7.1503,"2029.0000,2034.4615,2054.4615,2046.0000,2042.9231,2032.9231,2036.7692,2032.9231,2037.5385,2032.9231,2032.9231,2036.0000,2026.7692,2029.8462,2027.5385,2029.0769,2031.3846,2035.2308,2029.0769,2031.3846,2029.0769,2036.0000,2040.6154,2030.6154,2033.6923,2025.2308,2035.2308,2031.3846,2032.1538,2027.5385,2031.3846,2036.0000,2020.6154,2027.5385,2023.6923,2026.7692,2027.5385,2029.0769,2028.3077,2028.2308,2027.4615,2028.3077,2029.8462,2025.2308,2026.7692,2024.4615,2033.6923,2031.3846,2035.2308,2034.4615,2036.7692,2028.3077,2036.7692,2039.1538,2032.1538,2036.7692,2023.6923,2037.5385,2036.7692,2037.5385,2032.1538,2039.0769,2038.3077,2039.1538,2036.0000,2038.3077,2035.2308,2030.6154,2033.6923,2019.8462,2026.0000,2028.3077,2032.1538,2022.8462,2031.3846,2027.5385,2026.0000,2026.0000,2023.6923,2028.3077,2028.3077,2029.8462,2022.8462,2029.0000,2026.0000,2030.6154,2032.9231,2026.0000,2029.8462,2022.1538,2031.3846,2027.5385,2032.1538,2026.0000,2035.2308,2025.2308,2031.3846,2031.3846,2025.2308,2028.2308" +"benchmark intrusive graph dependency handling, N=1",creating nodes,100,5632,2252800,3.5572,3.4921,3.6611,0.4118,0.2923,0.5998,"3.3814,3.8848,3.4045,3.3814,3.8723,3.3796,3.3796,3.3796,3.3796,3.3814,3.9222,3.3814,3.3814,3.3814,3.3814,3.3814,3.8510,3.3796,3.3814,3.3885,3.3814,3.3814,3.8599,3.3814,3.3814,3.3814,3.3816,3.5112,5.8008,4.5075,4.8899,4.8899,4.8544,5.0359,3.4846,3.4009,3.3796,3.3796,3.3814,3.3814,3.3814,3.3814,3.8652,3.3814,3.3814,3.3814,3.3814,3.3796,3.8510,3.4348,3.3796,3.3991,3.3814,3.3796,3.7958,3.5167,3.3814,3.3814,3.3814,3.3991,3.6660,3.5753,3.3796,3.3796,3.3814,3.3814,3.8990,3.4029,3.3796,3.3796,3.3796,3.3814,3.9471,3.3796,3.3814,3.3991,3.3796,3.3796,3.8599,3.3814,3.3814,3.3814,3.3814,3.6625,3.6003,3.3796,3.3814,3.3814,3.3814,3.8919,3.3796,3.3885,3.3796,3.3796,3.3796,3.8617,3.3814,3.3814,3.3814,3.3814" +"benchmark intrusive graph dependency handling, N=1",creating and adding dependencies,100,1014,2433600,24.7500,24.6750,25.1065,0.7202,0.0408,1.7160,"24.6795,24.6302,24.8669,24.7199,24.6400,24.6607,24.6400,24.6696,24.7396,24.6795,24.7298,24.7189,24.6795,24.6598,24.6302,31.9024,24.7298,24.6795,24.6410,24.6598,24.7485,24.6795,24.7584,24.6607,24.7781,24.6410,24.6598,24.6598,24.7495,24.6696,24.7396,24.6795,24.7396,24.6400,24.6598,24.6903,24.7189,24.6302,24.6598,24.6400,24.6598,24.7189,24.6992,24.7298,24.6893,24.6410,24.6598,24.6302,24.6607,24.7485,24.6607,24.6598,24.6400,24.7692,24.6400,24.6499,24.6312,24.6598,24.6302,24.6607,24.6302,24.6499,24.6312,24.6598,24.6992,24.7091,24.6302,24.6607,24.6302,24.6598,24.6410,24.6696,24.7288,24.6696,24.7387,24.6706,24.7485,24.6706,24.6598,24.6499,24.7594,24.6400,24.6598,24.6302,24.6598,24.6706,24.7485,24.6302,24.6607,24.6400,24.6795,24.6903,24.7091,24.6410,24.6598,24.6302,24.6607,24.6302,24.6598,24.7298" +"benchmark intrusive graph dependency handling, N=1",adding and removing dependencies,100,1265,2403500,20.5968,20.5370,20.8916,0.5885,0.0105,1.4041,"20.5273,20.5510,20.5668,20.5589,20.5273,20.5352,20.5273,20.5431,20.5273,20.5431,20.5273,20.5431,20.5273,20.5431,20.5273,20.5431,20.5273,20.5273,20.5439,20.5273,20.5431,20.5273,20.5510,20.5281,20.5510,20.5273,20.5510,20.5273,20.5518,20.5273,20.5510,20.5273,20.5431,20.5281,20.5431,20.5273,20.5431,20.5273,20.5439,20.5273,20.5431,20.5273,20.5431,20.5281,20.5431,20.5273,20.5510,20.5273,20.5439,20.5273,20.5431,20.5273,20.5431,20.5281,20.5510,20.5273,20.5510,20.5273,20.5518,20.5273,20.5510,20.5273,20.5510,20.5273,20.5273,20.5431,20.5273,20.5431,20.5273,20.5431,20.5273,20.5510,20.5273,20.5510,20.5273,20.5510,20.5273,20.5510,20.5273,20.5439,20.5273,20.5510,20.5273,20.5510,20.5281,20.5431,20.5273,20.5510,26.4514,20.5510,20.5273,20.5510,20.5273,20.5518,20.5273,20.5510,20.5273,20.5510,20.5273,20.5510" +"benchmark intrusive graph dependency handling, N=1",checking for dependencies,100,10544,2108800,2.4190,2.4181,2.4200,0.0049,0.0047,0.0050,"2.4143,2.4247,2.4171,2.4248,2.4152,2.4143,2.4237,2.4144,2.4247,2.4152,2.4248,2.4143,2.4143,2.4247,2.4152,2.4247,2.4143,2.4248,2.4152,2.4247,2.4144,2.4143,2.4237,2.4152,2.4248,2.4152,2.4237,2.4144,2.4247,2.4152,2.4143,2.4237,2.4143,2.4248,2.4152,2.4247,2.4143,2.4144,2.4237,2.4152,2.4248,2.4143,2.4247,2.4152,2.4248,2.4143,2.4143,2.4248,2.4152,2.4247,2.4144,2.4237,2.4152,2.4248,2.4143,2.4143,2.4247,2.4153,2.4247,2.4143,2.4238,2.4143,2.4143,2.4247,2.4152,2.4247,2.4143,2.4238,2.4143,2.4247,2.4153,2.4152,2.4247,2.4143,2.4248,2.4152,2.4237,2.4144,2.4247,2.4152,2.4152,2.4237,2.4143,2.4248,2.4152,2.4247,2.4143,2.4144,2.4237,2.4143,2.4248,2.4152,2.4247,2.4143,2.4248,2.4152,2.4152,2.4248,2.4143,2.4237" +"benchmark intrusive graph dependency handling, N=10",creating nodes,100,516,2476800,44.3670,44.2316,44.9768,1.2649,0.1414,2.9995,"44.4031,43.9535,44.3624,44.0717,44.4012,43.9554,44.1880,44.2655,44.0523,44.4399,44.1880,44.1686,44.2849,44.0136,44.2074,44.1880,44.1318,44.4399,44.0136,44.4399,44.2267,44.2074,44.4787,44.1899,44.4399,44.2849,44.1105,44.3643,43.9942,44.4593,44.3062,44.2074,44.6143,44.4419,44.2074,44.2461,44.2074,44.3818,43.9748,44.1880,44.3430,44.0717,44.4012,43.9554,44.2655,44.3043,44.0523,44.6725,43.9554,44.1492,44.4419,44.1880,44.2074,44.4419,44.2655,44.3236,44.2074,44.1899,44.2267,44.1298,44.3430,44.2093,44.1880,44.2267,44.2074,44.3643,44.2074,44.4399,44.3236,44.0911,44.3818,44.2074,44.2074,44.2674,44.1880,44.4012,44.1880,44.1880,44.3818,44.0523,44.4225,43.9554,44.1686,44.2849,44.2074,44.2074,44.1880,44.2074,44.3043,44.2074,44.1899,44.2267,44.1105,44.3430,43.9942,44.2093,44.2267,56.8663,44.5581,44.2655" +"benchmark intrusive graph dependency handling, N=10",creating and adding dependencies,100,91,2520700,249.6458,248.9953,252.6251,6.0697,0.5697,14.4008,"248.5824,248.6923,254.0879,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.6813,249.7912,249.7912,249.7912,249.6813,249.7912,249.7912,249.7912,249.6813,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.7912,249.6813,249.7912,309.5824,248.5714,248.5714,248.6923,248.5824,248.5824,248.5824,248.5824,248.6813,248.5824,248.6923,248.5824,248.5824,248.5824,248.5824,248.6813,248.5824,248.6923,248.5824,248.5824,248.5824,248.5714,248.5714,248.5824,248.6923,248.5824,248.5824,248.5824,248.5714,248.6923,248.5824,248.6923,248.5824,248.5824,248.5714,248.5714,248.5824,248.5824,248.6923,248.5824,248.5824,248.5714,248.6923,248.5824,248.5824,248.5824,248.5824,248.6813,248.5714,248.5824,248.5824,248.5824,248.5824,248.6923,248.5714,248.5824,248.5824,248.6923,248.5824,248.5824,248.5714,248.5714,248.6923,248.5824,248.6923" +"benchmark intrusive graph dependency handling, N=10",adding and removing dependencies,100,104,2516800,241.4199,241.0465,243.0704,3.4977,0.3824,8.2964,"241.2115,240.7212,241.7788,241.3942,241.1154,240.7212,241.3942,240.8173,241.6923,241.3942,241.7885,240.8173,241.2115,241.3942,240.7308,241.7788,241.0192,241.2981,240.5385,241.1058,240.9231,240.9135,241.4038,240.7212,241.5000,241.0096,241.1154,240.3365,240.6250,242.5577,240.5288,241.7885,240.8173,241.9712,241.2115,240.6250,241.3077,240.5288,241.2981,240.8173,241.2981,241.0096,241.2019,240.4423,241.2981,241.1154,241.2981,240.6346,241.6827,240.9231,241.2981,240.5385,241.1058,240.6346,241.2981,240.7308,241.2019,240.6250,241.2981,240.5288,241.3077,241.3942,240.5288,241.3942,240.5385,241.2981,241.1154,241.3942,241.0192,241.2019,240.5385,241.2019,240.6346,241.2019,240.5288,241.2019,240.7212,241.1154,240.7212,241.4038,240.5288,241.2115,240.5288,241.3077,240.6250,241.7885,241.4904,275.9904,241.3942,240.6346,241.2019,240.6250,241.3077,240.9135,241.2115,240.7212,241.3077,240.7212,241.2115,240.7212" +"benchmark intrusive graph dependency handling, N=10",checking for dependencies,100,736,2502400,36.0539,36.0385,36.1036,0.1278,0.0483,0.2666,"35.9755,36.0707,37.2283,36.0571,36.0842,35.9891,36.0720,36.0842,35.9891,36.0842,35.9769,36.0842,36.0842,35.9755,36.0856,35.9755,36.0978,36.0707,35.9891,36.0707,35.9891,36.0707,36.0842,35.9891,36.0842,36.0842,35.9755,36.0992,35.9891,36.0842,36.0842,35.9769,36.0707,35.9891,36.0842,36.0720,35.9891,36.0842,35.9891,36.0707,36.0842,35.9755,36.0842,36.0842,35.9755,36.0842,35.9755,36.0842,36.0856,35.9755,36.0842,35.9755,36.0842,36.0707,35.9891,36.0842,35.9755,36.0856,36.0707,35.9891,36.0707,36.0856,35.9755,36.0842,35.9755,36.0856,36.0842,35.9755,36.0842,35.9755,36.0856,36.0707,35.9891,36.0707,35.9769,36.0842,36.0842,35.9755,36.0856,36.0707,35.9755,36.0842,35.9755,36.0707,36.0842,35.9755,36.0842,35.9755,36.0707,36.0842,35.9755,36.0720,35.9891,36.0842,36.0707,35.9755,36.0842,35.9755,36.0842,36.0842" +"benchmark intrusive graph dependency handling, N=100",creating nodes,100,59,2507500,428.6753,428.4614,429.2959,1.7217,0.7399,3.7214,"427.8983,429.0847,444.0339,428.7458,427.2203,429.2373,429.2373,428.9153,427.2203,428.7458,428.5763,428.7458,427.2203,429.2373,429.0678,429.0847,427.3898,428.7458,428.7458,428.9153,427.2203,429.0847,429.0847,429.2542,427.2034,428.7458,428.9153,429.2542,427.2203,429.2542,429.0847,428.9153,427.2203,428.9153,428.5593,429.0678,427.2034,429.4237,429.0847,429.0847,427.2203,428.7458,428.5763,428.7288,427.2034,429.0847,429.0847,429.0847,427.2203,429.2542,428.7458,428.7458,427.2034,429.0847,428.7458,429.2542,427.2203,429.0847,428.9153,429.2542,427.3729,428.8983,428.9153,429.0847,427.2203,429.2542,429.0847,428.7458,427.2203,428.7458,428.5593,428.5763,427.3898,429.2542,429.2542,429.0847,427.2203,428.7458,428.7288,428.7288,427.2203,429.0847,429.0847,429.0847,427.2203,429.0847,429.0847,428.7458,427.2034,428.5763,428.7458,428.9153,427.2203,429.0847,429.0847,429.0847,427.3729,428.9153,428.7458,427.5593" +"benchmark intrusive graph dependency handling, N=100",creating and adding dependencies,100,6,2946600,4942.3567,4928.3767,4994.6417,120.3335,24.4363,274.4672,"4905.6667,4915.6667,5242.8333,4944.0000,4925.6667,4949.0000,4957.3333,4940.6667,4944.0000,4930.6667,4942.3333,4935.6667,4909.0000,4939.0000,4932.3333,4932.3333,4950.6667,4932.1667,4945.6667,4914.0000,4919.0000,4929.0000,4932.3333,4927.3333,4952.3333,4939.0000,4925.6667,4879.0000,4935.6667,4930.6667,4915.5000,4954.0000,4955.6667,4930.6667,4935.6667,4970.8333,4935.6667,4900.5000,4947.3333,4955.6667,6076.1667,4994.1667,4920.5000,4920.6667,4937.3333,4947.5000,4955.6667,4929.0000,4957.3333,4939.0000,4932.3333,4960.8333,4950.6667,4962.3333,4932.3333,4950.8333,4920.5000,4944.0000,4962.3333,4949.0000,4942.3333,4944.1667,4937.3333,4957.3333,4925.6667,4967.3333,4930.6667,4922.1667,4923.8333,4914.0000,4894.0000,4877.1667,4889.0000,4912.3333,4883.8333,4907.3333,4912.3333,4929.0000,4908.8333,4875.6667,4935.6667,4903.8333,4898.8333,4924.0000,4924.0000,4892.3333,4897.1667,4880.5000,4909.0000,4910.5000,4902.1667,4900.6667,4924.0000,4932.3333,4912.3333,4893.8333,4922.3333,4920.6667,4890.6667,4903.8333" +"benchmark intrusive graph dependency handling, N=100",adding and removing dependencies,100,6,2935800,5042.9017,5033.6633,5064.6833,68.1540,35.1009,140.5218,"5092.6667,5009.1667,5027.5000,5014.1667,5052.5000,5015.8333,5610.3333,5061.0000,5015.8333,5052.5000,4985.8333,5084.1667,5035.8333,5035.8333,4974.0000,4969.1667,5014.1667,5057.5000,5055.8333,5024.1667,5075.8333,4979.1667,5069.1667,5057.6667,5010.8333,5075.8333,5074.3333,4990.8333,5019.1667,5019.1667,5091.0000,5034.1667,5030.8333,5059.1667,5014.1667,5066.0000,5032.5000,5072.6667,5002.3333,5064.1667,5039.1667,5035.8333,5067.6667,5027.5000,5077.6667,5062.5000,5020.8333,5061.0000,4982.3333,5044.3333,4990.6667,5005.8333,5034.1667,5056.0000,5019.1667,5049.1667,5037.5000,5082.6667,5101.0000,5009.1667,4910.6667,5067.5000,5081.0000,5066.0000,5029.1667,5037.5000,5004.1667,5050.8333,5084.3333,5059.1667,5037.6667,5027.5000,5042.5000,5062.5000,5024.0000,5082.5000,5037.5000,5070.8333,5050.8333,5049.3333,5039.1667,5039.1667,5059.3333,5045.8333,5079.3333,5062.5000,5069.3333,5065.8333,5032.6667,4957.3333,4902.3333,4917.1667,5092.6667,5032.5000,5057.6667,5050.8333,5025.8333,4995.8333,5055.8333,5002.3333" +"benchmark intrusive graph dependency handling, N=100",checking for dependencies,100,14,2599800,1913.3486,1908.1900,1935.7564,47.3790,6.8932,111.8945,"1912.7143,1919.2143,1948.5000,1917.7857,1905.6429,1909.1429,1915.6429,1914.9286,1902.0000,1912.0714,1895.5714,1906.3571,1900.5714,1898.4286,1908.4286,1910.6429,1906.2857,1909.2143,1907.7143,1901.3571,1914.2143,1900.5714,1912.7857,1902.0000,1897.7143,1902.7857,1896.2857,1908.5000,1908.4286,1907.7857,1909.9286,1909.1429,1907.0714,1902.0000,1905.6429,1909.1429,1905.6429,1908.4286,1907.0714,1909.2143,1907.7143,1913.5000,1897.7143,1904.9286,1911.2857,1907.7857,1909.2143,1906.2857,1908.5000,1904.8571,1912.7857,1902.0000,1902.7857,1899.1429,1907.7857,2378.6429,1914.1429,1908.4286,1903.5000,1906.2857,1907.0714,1900.5714,1897.0000,1903.5000,1896.2857,1906.3571,1908.4286,1907.7857,1903.4286,1903.5000,1913.4286,1899.2143,1902.0000,1896.2857,1907.0714,1909.2143,1909.1429,1912.7857,1896.2857,1903.5000,1909.8571,1907.7857,1908.4286,1907.7143,1915.6429,1921.3571,1922.7857,1916.3571,1923.5000,1913.4286,1918.5000,1912.7857,1919.2143,1912.7143,1919.1429,1919.2143,1915.6429,1919.9286,1914.9286,1919.2143" diff --git a/ci/perf/gpuc1_bench.md b/ci/perf/gpuc1_bench.md new file mode 100644 index 000000000..30bd4ffb1 --- /dev/null +++ b/ci/perf/gpuc1_bench.md @@ -0,0 +1,23 @@ +# Benchmark Results + +| Metadata | | +| :------- | :------------------- | +| Created | 2022-03-29T13:49:18Z | + + +| Test case | Benchmark name | Mean | Std dev | +| :--------------------------------------------------- | :------------------------------- | -------: | ------: | +| benchmark intrusive graph dependency handling, N=1 | creating nodes | 3.56 | 0.41 | +| benchmark intrusive graph dependency handling, N=1 | creating and adding dependencies | 24.75 | 0.72 | +| benchmark intrusive graph dependency handling, N=1 | adding and removing dependencies | 20.60 | 0.59 | +| benchmark intrusive graph dependency handling, N=1 | checking for dependencies | 2.42 | 0.00 | +| benchmark intrusive graph dependency handling, N=10 | creating nodes | 44.37 | 1.26 | +| benchmark intrusive graph dependency handling, N=10 | creating and adding dependencies | 249.65 | 6.07 | +| benchmark intrusive graph dependency handling, N=10 | adding and removing dependencies | 241.42 | 3.50 | +| benchmark intrusive graph dependency handling, N=10 | checking for dependencies | 36.05 | 0.13 | +| benchmark intrusive graph dependency handling, N=100 | creating nodes | 428.68 | 1.72 | +| benchmark intrusive graph dependency handling, N=100 | creating and adding dependencies | 4'942.36 | 120.33 | +| benchmark intrusive graph dependency handling, N=100 | adding and removing dependencies | 5'042.90 | 68.15 | +| benchmark intrusive graph dependency handling, N=100 | checking for dependencies | 1'913.35 | 47.38 | + +All numbers are in nanoseconds. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0efe3d365..423716975 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -68,7 +68,7 @@ target_link_libraries(all_tests PRIVATE test_main) set_test_target_parameters(all_tests "") # Unit benchmark executable -add_executable(benchmarks benchmarks.cc benchmark_csv_reporter.cc) +add_executable(benchmarks benchmarks.cc benchmark_reporters.cc) target_link_libraries(benchmarks PRIVATE test_main) set_test_target_parameters(benchmarks benchmarks.cc) diff --git a/test/benchmark_csv_reporter.cc b/test/benchmark_csv_reporter.cc deleted file mode 100644 index c0d2a9c1e..000000000 --- a/test/benchmark_csv_reporter.cc +++ /dev/null @@ -1,93 +0,0 @@ -#include -#include - -#include -#include -#include -#include -#include - -// Escape according to RFC 4180 (w/o line break support) -static std::string escape_csv(const std::string& str) { - assert(str.find_first_of("\r\n") == std::string::npos); - // Determine if we need escaping at all (string contains comma or quote). - if(str.find_first_of("\",") == std::string::npos) return str; - - const std::regex re("\""); - // Prefix quotes with a another quote, wrap whole string in quotes. - return fmt::format("\"{}\"", std::regex_replace(str, re, "\"\"")); -} - -class benchmark_reporter_base : public Catch::StreamingReporterBase { - public: - using StreamingReporterBase::StreamingReporterBase; - - // TODO: Do we want to somehow report this? - void benchmarkFailed(Catch::StringRef benchmarkName) override { StreamingReporterBase::benchmarkFailed(benchmarkName); } - - void sectionStarting(Catch::SectionInfo const& sectionInfo) override { - StreamingReporterBase::sectionStarting(sectionInfo); - // Each test case has an implicit section with the name of the test case itself, - // so there is no need to capture that separately. - active_sections.push_back(sectionInfo.name); - } - - void testCasePartialEnded(Catch::TestCaseStats const& testCaseStats, uint64_t partNumber) override { - StreamingReporterBase::testCasePartialEnded(testCaseStats, partNumber); - // If the exact same set of sections was active as before, generators must have been involved. - if(active_sections == previous_active_sections) { - if(!did_print_generators_warning) { - fmt::print("WARNING: Using generators will result in indistinguishable test case columns.\n"); - did_print_generators_warning = true; - } - } - std::swap(active_sections, previous_active_sections); - active_sections.clear(); - } - - protected: - std::string get_test_case_name() const { return fmt::format("{}", fmt::join(active_sections, " > ")); } - - private: - std::vector active_sections; - std::vector previous_active_sections; - bool did_print_generators_warning = false; -}; - -/** - * Prints benchmark results in CSV format. - * All timings are in nanoseconds. - * - * Note that unlike in some other reporters, sections that precede a BENCHMARK and are - * active in the current invocation will be printed as well (not just those directly - * surrounding a BENCHMARK). - */ -class benchmark_csv_reporter : public benchmark_reporter_base { - public: - using benchmark_reporter_base::benchmark_reporter_base; - - static std::string getDescription() { return "Reporter for benchmarks in CSV format"; } - - void testRunStarting(Catch::TestRunInfo const& testRunInfo) override { - StreamingReporterBase::testRunStarting(testRunInfo); - fmt::print(m_stream, "test case,benchmark name,samples,iterations,estimated,mean,low mean,high mean,std dev,low std dev,high std dev,raw\n"); - } - - void benchmarkEnded(Catch::BenchmarkStats<> const& benchmarkStats) override { - StreamingReporterBase::benchmarkEnded(benchmarkStats); - auto& info = benchmarkStats.info; - fmt::print(m_stream, "{},{},{},{},{},", escape_csv(get_test_case_name()), escape_csv(info.name), info.samples, info.iterations, info.estimatedDuration); - fmt::print(m_stream, "{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},", benchmarkStats.mean.point.count(), benchmarkStats.mean.lower_bound.count(), - benchmarkStats.mean.upper_bound.count(), benchmarkStats.standardDeviation.point.count(), benchmarkStats.standardDeviation.lower_bound.count(), - benchmarkStats.standardDeviation.upper_bound.count()); - // Finally print all raw values for custom analyses (as quoted comma-separated values). - std::vector raw; - raw.reserve(benchmarkStats.samples.size()); - std::transform(benchmarkStats.samples.cbegin(), benchmarkStats.samples.cend(), std::back_inserter(raw), [](auto& d) { return d.count(); }); - fmt::print(m_stream, "\"{:.4f}\"\n", fmt::join(raw, ",")); - // Flush so we can watch results come in when writing to file - m_stream.flush(); - } -}; - -CATCH_REGISTER_REPORTER("celerity-benchmark-csv", benchmark_csv_reporter) diff --git a/test/benchmark_reporters.cc b/test/benchmark_reporters.cc new file mode 100644 index 000000000..c92eaa9ae --- /dev/null +++ b/test/benchmark_reporters.cc @@ -0,0 +1,219 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +// Escape according to RFC 4180 (w/o line break support) +static std::string escape_csv(const std::string& str) { + assert(str.find_first_of("\r\n") == std::string::npos); + // Determine if we need escaping at all (string contains comma or quote). + if(str.find_first_of("\",") == std::string::npos) return str; + + const std::regex re("\""); + // Prefix quotes with a another quote, wrap whole string in quotes. + return fmt::format("\"{}\"", std::regex_replace(str, re, "\"\"")); +} + +class benchmark_reporter_base : public Catch::StreamingReporterBase { + public: + using StreamingReporterBase::StreamingReporterBase; + + // TODO: Do we want to somehow report this? + void benchmarkFailed(Catch::StringRef benchmarkName) override { StreamingReporterBase::benchmarkFailed(benchmarkName); } + + void sectionStarting(Catch::SectionInfo const& sectionInfo) override { + StreamingReporterBase::sectionStarting(sectionInfo); + // Each test case has an implicit section with the name of the test case itself, + // so there is no need to capture that separately. + active_sections.push_back(sectionInfo.name); + } + + void testCasePartialEnded(Catch::TestCaseStats const& testCaseStats, uint64_t partNumber) override { + StreamingReporterBase::testCasePartialEnded(testCaseStats, partNumber); + // If the exact same set of sections was active as before, generators must have been involved. + if(active_sections == previous_active_sections) { + if(!did_print_generators_warning) { + fmt::print("WARNING: Using generators will result in indistinguishable test case columns.\n"); + did_print_generators_warning = true; + } + } + std::swap(active_sections, previous_active_sections); + active_sections.clear(); + } + + protected: + std::string get_test_case_name() const { return fmt::format("{}", fmt::join(active_sections, " > ")); } + + private: + std::vector active_sections; + std::vector previous_active_sections; + bool did_print_generators_warning = false; +}; + +/** + * Prints benchmark results in CSV format. + * All timings are in nanoseconds. + * + * Note that unlike in some other reporters, sections that precede a BENCHMARK and are + * active in the current invocation will be printed as well (not just those directly + * surrounding a BENCHMARK). + */ +class benchmark_csv_reporter : public benchmark_reporter_base { + public: + using benchmark_reporter_base::benchmark_reporter_base; + + static std::string getDescription() { return "Reporter for benchmarks in CSV format"; } + + void testRunStarting(Catch::TestRunInfo const& testRunInfo) override { + StreamingReporterBase::testRunStarting(testRunInfo); + fmt::print(m_stream, "test case,benchmark name,samples,iterations,estimated,mean,low mean,high mean,std dev,low std dev,high std dev,raw\n"); + } + + void benchmarkEnded(Catch::BenchmarkStats<> const& benchmarkStats) override { + StreamingReporterBase::benchmarkEnded(benchmarkStats); + auto& info = benchmarkStats.info; + fmt::print(m_stream, "{},{},{},{},{},", escape_csv(get_test_case_name()), escape_csv(info.name), info.samples, info.iterations, info.estimatedDuration); + fmt::print(m_stream, "{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},", benchmarkStats.mean.point.count(), benchmarkStats.mean.lower_bound.count(), + benchmarkStats.mean.upper_bound.count(), benchmarkStats.standardDeviation.point.count(), benchmarkStats.standardDeviation.lower_bound.count(), + benchmarkStats.standardDeviation.upper_bound.count()); + // Finally print all raw values for custom analyses (as quoted comma-separated values). + std::vector raw; + raw.reserve(benchmarkStats.samples.size()); + std::transform(benchmarkStats.samples.cbegin(), benchmarkStats.samples.cend(), std::back_inserter(raw), [](auto& d) { return d.count(); }); + fmt::print(m_stream, "\"{:.4f}\"\n", fmt::join(raw, ",")); + // Flush so we can watch results come in when writing to file + m_stream.flush(); + } +}; + +CATCH_REGISTER_REPORTER("celerity-benchmark-csv", benchmark_csv_reporter) + +enum class align : char { left = '<', right = '>', center = '^' }; + +class markdown_table_printer { + struct column { + static constexpr size_t min_width = 3; + std::string header; + size_t width; + align alignment; + }; + + public: + markdown_table_printer(const std::vector>& columns) { + std::transform(columns.cbegin(), columns.end(), std::back_inserter(this->columns), [](const auto& c) { + const auto [name, alignment] = c; + return column{name, std::max(column::min_width, name.length()), alignment}; + }); + } + + void add_row(std::vector cells) { + if(cells.size() != columns.size()) { throw std::runtime_error("Column mismatch"); } + for(size_t i = 0; i < columns.size(); ++i) { + columns[i].width = std::max(columns[i].width, cells[i].length()); + } + rows.push_back(std::move(cells)); + } + + void print(std::ostream& os) const { + // fmt does not allow to set alignment dynamically, so we need a helper function. + // Replaces 'A' in fmt_str with '<','>' or '^'. + constexpr auto align_fmt = [](std::string fmt_str, align a) { + std::replace(fmt_str.begin(), fmt_str.end(), 'A', static_cast(a)); + return fmt_str; + }; + + // Print column headers + fmt::print(os, "|"); + for(const auto& [header, width, a] : columns) { + fmt::print(os, align_fmt(" {: A{}} |", a), header, width); + } + fmt::print(os, "\n"); + + // Print separators + fmt::print(os, "|"); + for(const auto& [_, width, a] : columns) { + const char align_left = a != align::right ? ':' : '-'; + const char align_right = a != align::left ? ':' : '-'; + fmt::print(os, align_fmt(" {}{:-A{}}{} |", a), align_left, "", width - 2, align_right); + } + fmt::print(os, "\n"); + + // Print rows + for(const auto& r : rows) { + fmt::print(os, "|"); + for(size_t i = 0; i < r.size(); ++i) { + fmt::print(os, align_fmt(" {: A{}} |", columns[i].alignment), r[i], columns[i].width); + } + fmt::print(os, "\n"); + } + } + + private: + std::vector columns; + std::vector> rows; +}; + +class benchmark_md_reporter : public benchmark_reporter_base { + public: + using benchmark_reporter_base::benchmark_reporter_base; + + static std::string getDescription() { return "Generates a Markdown report for benchmark results"; } + + void testRunStarting(Catch::TestRunInfo const& testRunInfo) override { + StreamingReporterBase::testRunStarting(testRunInfo); + + fmt::print(m_stream, "# Benchmark Results\n\n"); + + markdown_table_printer meta_printer({{"Metadata", align::left}, {"", align::left}}); + const auto now_gmt = fmt::gmtime(std::time(nullptr)); + // TODO: It would be cool to also have celerity version, hostname, argv, ... + meta_printer.add_row({"Created", fmt::format("{:%FT%TZ}", now_gmt)}); + + meta_printer.print(m_stream); + } + + void testRunEnded(Catch::TestRunStats const& testRunStats) override { + StreamingReporterBase::testRunEnded(testRunStats); + fmt::print(m_stream, "\n\n"); + results_printer.print(m_stream); + fmt::print(m_stream, "\nAll numbers are in nanoseconds.\n"); + } + + void benchmarkEnded(Catch::BenchmarkStats<> const& benchmarkStats) override { + StreamingReporterBase::benchmarkEnded(benchmarkStats); + + // Format numbers with ' as thousand separator and . as decimal separator. + constexpr auto format_result = [](std::chrono::duration ns) { + // fmt can only do thousands separators based on locale, so we need to do a character replacement afterwards. + // Also it only works on integral types, so we need to format the fractional part separately. + double integral; + const double fractional = std::modf(ns.count(), &integral); + auto integral_formatted = fmt::format(std::locale("en_US.UTF-8"), "{:L}", static_cast(integral)); + std::replace(integral_formatted.begin(), integral_formatted.end(), ',', '\''); + const auto fractional_formatted = fmt::format("{:.2f}", fractional).substr(2); + return fmt::format("{}.{}", integral_formatted, fractional_formatted); + }; + + results_printer.add_row({fmt::format("{}", get_test_case_name()), // Test case + benchmarkStats.info.name, // Benchmark name + format_result(benchmarkStats.mean.point), // Mean + format_result(benchmarkStats.standardDeviation.point)}); // Std dev + } + + private: + markdown_table_printer results_printer{{{"Test case", align::left}, {"Benchmark name", align::left}, {"Mean", align::right}, {"Std dev", align::right}}}; +}; + +CATCH_REGISTER_REPORTER("celerity-benchmark-md", benchmark_md_reporter)