Version-sensitive plutus-benchmark tests (PLT 7544) (#5587)

* Reorganise nofib tests

* Versioned golden tests for nofib

* Realign comment

* Versioned golden tests for marlowe

* Versioned golden tests for lists and script contexts

* Versioned golden tests for ed25519 and bls12-381 costs

* Tidying up

* Update some golden files

* Update some golden files

* Tidy up directory path usage

* Comment about directory structure

plutus-benchmark/bls12-381-costs/test/9.6/bls12-381-costs.golden

@@ -0,0 +1,152 @@
+Hash n bytestrings onto G1 and add points
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    0      77   (0.5%)             100   (0.0%)             100   (0.0%) 
+   10     187   (1.1%)       681707800   (6.8%)           39342   (0.3%) 
+   20     297   (1.8%)      1363794920  (13.6%)           75702   (0.5%) 
+   30     407   (2.5%)      2045882040  (20.5%)          112062   (0.8%) 
+   40     517   (3.2%)      2727969160  (27.3%)          148422   (1.1%) 
+   50     627   (3.8%)      3410056280  (34.1%)          184782   (1.3%) 
+   60     737   (4.5%)      4092143400  (40.9%)          221142   (1.6%) 
+   70     847   (5.2%)      4774230520  (47.7%)          257502   (1.8%) 
+   80     957   (5.8%)      5456317640  (54.6%)          293862   (2.1%) 
+   90    1067   (6.5%)      6138404760  (61.4%)          330222   (2.4%) 
+  100    1177   (7.2%)      6820491880  (68.2%)          366582   (2.6%) 
+  110    1287   (7.9%)      7502579000  (75.0%)          402942   (2.9%) 
+  120    1397   (8.5%)      8184666120  (81.8%)          439302   (3.1%) 
+  130    1507   (9.2%)      8866753240  (88.7%)          475662   (3.4%) 
+  140    1617   (9.9%)      9548840360  (95.5%)          512022   (3.7%) 
+  150    1727  (10.5%)     10230927480 (102.3%)          548382   (3.9%) 
+
+
+Hash n bytestrings onto G2 and add points
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    0      77   (0.5%)             100   (0.0%)             100   (0.0%) 
+   10     187   (1.1%)      2075992430  (20.8%)           39684   (0.3%) 
+   20     297   (1.8%)      4153677170  (41.5%)           76404   (0.5%) 
+   30     407   (2.5%)      6231361910  (62.3%)          113124   (0.8%) 
+   40     517   (3.2%)      8309046650  (83.1%)          149844   (1.1%) 
+   50     627   (3.8%)     10386731390 (103.9%)          186564   (1.3%) 
+   60     737   (4.5%)     12464416130 (124.6%)          223284   (1.6%) 
+   70     847   (5.2%)     14542100870 (145.4%)          260004   (1.9%) 
+   80     957   (5.8%)     16619785610 (166.2%)          296724   (2.1%) 
+   90    1067   (6.5%)     18697470350 (187.0%)          333444   (2.4%) 
+  100    1177   (7.2%)     20775155090 (207.8%)          370164   (2.6%) 
+  110    1287   (7.9%)     22852839830 (228.5%)          406884   (2.9%) 
+  120    1397   (8.5%)     24930524570 (249.3%)          443604   (3.2%) 
+  130    1507   (9.2%)     27008209310 (270.1%)          480324   (3.4%) 
+  140    1617   (9.9%)     29085894050 (290.9%)          517044   (3.7%) 
+  150    1727  (10.5%)     31163578790 (311.6%)          553764   (4.0%) 
+
+
+Uncompress n G1 points and add the results
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    0      69   (0.4%)             100   (0.0%)             100   (0.0%) 
+   10     619   (3.8%)       183823250   (1.8%)           37042   (0.3%) 
+   20    1169   (7.1%)       368094820   (3.7%)           71402   (0.5%) 
+   30    1719  (10.5%)       552366390   (5.5%)          105762   (0.8%) 
+   40    2269  (13.8%)       736637960   (7.4%)          140122   (1.0%) 
+   50    2819  (17.2%)       920909530   (9.2%)          174482   (1.2%) 
+   60    3369  (20.6%)      1105181100  (11.1%)          208842   (1.5%) 
+   70    3919  (23.9%)      1289452670  (12.9%)          243202   (1.7%) 
+   80    4469  (27.3%)      1473724240  (14.7%)          277562   (2.0%) 
+   90    5019  (30.6%)      1657995810  (16.6%)          311922   (2.2%) 
+  100    5569  (34.0%)      1842267380  (18.4%)          346282   (2.5%) 
+  110    6119  (37.3%)      2026538950  (20.3%)          380642   (2.7%) 
+  120    6669  (40.7%)      2210810520  (22.1%)          415002   (3.0%) 
+  130    7219  (44.1%)      2395082090  (24.0%)          449362   (3.2%) 
+  140    7769  (47.4%)      2579353660  (25.8%)          483722   (3.5%) 
+  150    8319  (50.8%)      2763625230  (27.6%)          518082   (3.7%) 
+
+
+Uncompress n G2 points and add the results
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    0      69   (0.4%)             100   (0.0%)             100   (0.0%) 
+   10    1099   (6.7%)       361567910   (3.6%)           37384   (0.3%) 
+   20    2129  (13.0%)       724897130   (7.2%)           72104   (0.5%) 
+   30    3159  (19.3%)      1088226350  (10.9%)          106824   (0.8%) 
+   40    4189  (25.6%)      1451555570  (14.5%)          141544   (1.0%) 
+   50    5219  (31.9%)      1814884790  (18.1%)          176264   (1.3%) 
+   60    6249  (38.1%)      2178214010  (21.8%)          210984   (1.5%) 
+   70    7279  (44.4%)      2541543230  (25.4%)          245704   (1.8%) 
+   80    8309  (50.7%)      2904872450  (29.0%)          280424   (2.0%) 
+   90    9339  (57.0%)      3268201670  (32.7%)          315144   (2.3%) 
+  100   10369  (63.3%)      3631530890  (36.3%)          349864   (2.5%) 
+  110   11399  (69.6%)      3994860110  (39.9%)          384584   (2.7%) 
+  120   12429  (75.9%)      4358189330  (43.6%)          419304   (3.0%) 
+  130   13459  (82.1%)      4721518550  (47.2%)          454024   (3.2%) 
+  140   14489  (88.4%)      5084847770  (50.8%)          488744   (3.5%) 
+  150   15519  (94.7%)      5448176990  (54.5%)          523464   (3.7%) 
+
+
+Apply pairing to two pairs of points in G1 x G2 and run finalVerify on the results
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     334   (2.0%)      1193741374  (11.9%)            3746   (0.0%) 
+
+
+Groth16 verification example
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     779   (4.8%)      2316309514  (23.2%)           10904   (0.1%) 
+
+VRF example
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     747   (4.6%)      1324508370  (13.2%)          319515   (2.3%) 
+
+G1 Verify
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     325   (2.0%)      1448435500  (14.5%)            4936   (0.0%) 
+
+G2 Verify
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     374   (2.3%)      1310188206  (13.1%)            4918   (0.0%) 
+
+Aggregate Single Key
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     773   (4.7%)      3326501083  (33.3%)           70684   (0.5%) 
+
+Aggregate Multi Key
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -    1737  (10.6%)      3708397401  (37.1%)          565657   (4.0%) 
+
+Schnorr Signature G1
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     419   (2.6%)       534305472   (5.3%)          264506   (1.9%) 
+
+Schnorr Signature G2
+
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    -     563   (3.4%)       953950193   (9.5%)          264692   (1.9%) 
+
+Groth16Verify succeeded
+Simple Verify succeeded
+VRF succeeded
+G1 Verify succeeded
+G2 Verify succeeded
+Aggregate Signature Single Key G1 Verify succeeded
+Aggregate Signature Multi Key G2 Verify succeeded
+Schnorr G1 Verify succeeded
+Schnorr G2 Verify succeeded

plutus-benchmark/bls12-381-costs/test/Spec.hs

@@ -3,11 +3,13 @@ module Main where
 import PlutusBenchmark.BLS12_381.RunTests (runTests)
 import PlutusBenchmark.Common (checkGoldenFileExists, goldenVsTextualOutput)
 
-outputFile :: String
+import Test.Tasty.Extras (makeVersionedFilePath)
+
+outputFile :: FilePath
 outputFile = "bls12-381-costs.txt"
 
 goldenFile :: FilePath
-goldenFile = "bls12-381-costs/test/bls12-381-costs.golden"
+goldenFile = makeVersionedFilePath ["bls12-381-costs", "test"] "bls12-381-costs.golden"
 
 main :: IO ()
 main = do

plutus-benchmark/common/PlutusBenchmark/Common.hs

@@ -208,7 +208,7 @@ printSizeStatistics h n script = do
 goldenVsTextualOutput
     :: TestName          -- The name of the test.
     -> FilePath          -- The path to the golden file.
-    -> String            -- The name of the results file (may be extended to make it unique).
+    -> FilePath          -- The name of the results file (may be extended to make it unique).
     -> (Handle -> IO a)  -- A function which runs tests and writes output to the given handle.
     -> IO ()
 goldenVsTextualOutput testName goldenFile filename runTest =  do

plutus-benchmark/ed25519-costs/test/9.6/ed25519-costs.golden

@@ -0,0 +1,20 @@
+    n     Script size             CPU usage               Memory usage
+  ----------------------------------------------------------------------
+    0     496   (3.0%)         6713466   (0.1%)           24554   (0.2%) 
+   10    2223  (13.6%)       734905406   (7.3%)          519704   (3.7%) 
+   20    3950  (24.1%)      1463097346  (14.6%)         1014854   (7.2%) 
+   30    5677  (34.6%)      2191289286  (21.9%)         1510004  (10.8%) 
+   40    7404  (45.2%)      2919481226  (29.2%)         2005154  (14.3%) 
+   50    9130  (55.7%)      3647673166  (36.5%)         2500304  (17.9%) 
+   60   10857  (66.3%)      4375865106  (43.8%)         2995454  (21.4%) 
+   70   12584  (76.8%)      5104057046  (51.0%)         3490604  (24.9%) 
+   80   14310  (87.3%)      5832248986  (58.3%)         3985754  (28.5%) 
+   90   16037  (97.9%)      6560440926  (65.6%)         4480904  (32.0%) 
+  100   17764 (108.4%)      7288632866  (72.9%)         4976054  (35.5%) 
+  110   19491 (119.0%)      8016824806  (80.2%)         5471204  (39.1%) 
+  120   21217 (129.5%)      8745016746  (87.5%)         5966354  (42.6%) 
+  130   22944 (140.0%)      9473208686  (94.7%)         6461504  (46.2%) 
+  140   24671 (150.6%)     10201400626 (102.0%)         6956654  (49.7%) 
+  150   26398 (161.1%)     10929592566 (109.3%)         7451804  (53.2%) 
+
+Off-chain version succeeded on 100 inputs

plutus-benchmark/ed25519-costs/test/Spec.hs

@@ -3,11 +3,13 @@ module Main where
 import PlutusBenchmark.Common (checkGoldenFileExists, goldenVsTextualOutput)
 import PlutusBenchmark.Ed25519.Common (runTests)
 
-outputFile :: String
+import Test.Tasty.Extras (makeVersionedFilePath)
+
+outputFile :: FilePath
 outputFile = "ed25519-costs.txt"
 
 goldenFile :: FilePath
-goldenFile = "ed25519-costs/test/ed25519-costs.golden"
+goldenFile = makeVersionedFilePath ["ed25519-costs", "test"] "ed25519-costs.golden"
 
 main :: IO ()
 main = do

plutus-benchmark/lists/test/Sum/9.6/left-fold-built-in.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 174359654
+| mem: 564532})

plutus-benchmark/lists/test/Sum/9.6/left-fold-data.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 368280389
+| mem: 1114530})

plutus-benchmark/lists/test/Sum/9.6/left-fold-scott.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 132243800
+| mem: 485500})

plutus-benchmark/lists/test/Sum/9.6/right-fold-built-in.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 181259654
+| mem: 594532})

plutus-benchmark/lists/test/Sum/9.6/right-fold-data.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 375180389
+| mem: 1144530})

plutus-benchmark/lists/test/Sum/9.6/right-fold-scott.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 139143800
+| mem: 515500})

plutus-benchmark/lists/test/Sum/Spec.hs

@@ -2,7 +2,7 @@
 module Sum.Spec (tests) where
 
 import Test.Tasty
-import Test.Tasty.Extras
+import Test.Tasty.Extras (TestNested, runTestGroupNestedGhc)
 import Test.Tasty.QuickCheck
 
 import PlutusBenchmark.Common (Term, cekResultMatchesHaskellValue)
@@ -12,6 +12,11 @@ import PlutusBenchmark.Lists.Sum.HandWritten qualified as HandWritten
 
 import PlutusTx.Test qualified as Tx
 
+-- Make a set of golden tests with results stored in a given subdirectory
+-- inside a subdirectory determined by the GHC version.
+testGroupGhcIn :: [FilePath] -> [TestNested] -> TestTree
+testGroupGhcIn dir = runTestGroupNestedGhc (["lists", "test"] ++ dir)
+
 -- | Check that the various summation functions all give the same result as 'sum'
 
 prop_sum :: ([Integer] -> Term) -> [Integer] -> Property
@@ -32,7 +37,7 @@ tests =
       , testProperty "Compiled left fold (built-in lists)"     $ prop_sum Compiled.mkSumLeftBuiltinTerm
       , testProperty "Compiled left fold (data lists)"         $ prop_sum Compiled.mkSumLeftDataTerm
       ]
-    , runTestNestedIn ["lists", "test"] $ testNested "Sum"
+    , testGroupGhcIn ["Sum"]
       [ Tx.goldenBudget "right-fold-scott"    $ Compiled.mkSumRightScottCode input
       , Tx.goldenBudget "right-fold-built-in" $ Compiled.mkSumRightBuiltinCode input
       , Tx.goldenBudget "right-fold-data"     $ Compiled.mkSumRightDataCode input

plutus-benchmark/marlowe/test/Spec.hs

@@ -4,7 +4,7 @@
 module Main (main) where
 
 import Test.Tasty
-import Test.Tasty.Extras
+import Test.Tasty.Extras (TestNested, runTestGroupNestedGhc)
 
 import PlutusBenchmark.Marlowe.BenchUtil (benchmarkToUPLC, rolePayoutBenchmarks,
                                           semanticsBenchmarks)
@@ -28,6 +28,11 @@ mkBudgetTest validator bm@M.Benchmark{..} =
   in
     (benchName, benchmarkToUPLC validator bm)
 
+-- Make a set of golden tests with results stored in a given subdirectory
+-- inside a subdirectory determined by the GHC version.
+testGroupGhcIn :: [FilePath] -> [TestNested] -> TestTree
+testGroupGhcIn path = runTestGroupNestedGhc (["marlowe", "test"] ++ path)
+
 main :: IO ()
 main = do
 
@@ -40,13 +45,13 @@ main = do
   let allTests :: TestTree
       allTests =
         testGroup "plutus-benchmark Marlowe tests"
-            [ runTestNestedIn ["marlowe", "test"] $ testNested "semantics" $
+            [ testGroupGhcIn ["semantics"] $
                 goldenSize "semantics" marloweValidator
                   : [ goldenUEvalBudget name [value]
                     | bench <- semanticsMBench
                     , let (name, value) = mkBudgetTest marloweValidator bench
                     ]
-            , runTestNestedIn ["marlowe", "test"] $ testNested "role-payout" $
+            , testGroupGhcIn ["role-payout"] $
                 goldenSize "role-payout" rolePayoutValidator
                   : [ goldenUEvalBudget name [value]
                     | bench <- rolePayoutMBench

plutus-benchmark/marlowe/test/role-payout/9.6/0004000402010401030101030100040000010104020201030001000204020401.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 275022771
+| mem: 977108})

plutus-benchmark/marlowe/test/role-payout/9.6/0100000100010000000001000100010101000101000001000000010000010000.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 405186590
+| mem: 1501014})

plutus-benchmark/marlowe/test/role-payout/9.6/0101000100000101010000010101000100010101000001000001000000010101.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 296920997
+| mem: 1066908})

plutus-benchmark/marlowe/test/role-payout/9.6/01dcc372ea619cb9f23c45b17b9a0a8a16b7ca0e04093ef8ecce291667a99a4c.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 239421811
+| mem: 842676})

plutus-benchmark/marlowe/test/role-payout/9.6/0201020201020000020000010201020001020200000002010200000101010100.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 268525940
+| mem: 947250})

plutus-benchmark/marlowe/test/role-payout/9.6/0202010002010100020102020102020001010101020102010001010101000100.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 247383894
+| mem: 868686})

plutus-benchmark/marlowe/test/role-payout/9.6/0303020000020001010201060303040208070100050401080304020801030001.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 245568317
+| mem: 862084})

plutus-benchmark/marlowe/test/role-payout/9.6/031d56d71454e2c4216ffaa275c4a8b3eb631109559d0e56f44ea8489f57ba97.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 311520242
+| mem: 1115262})

plutus-benchmark/marlowe/test/role-payout/9.6/03d730a62332c51c7b70c16c64da72dd1c3ea36c26b41cd1a1e00d39fda3d6cc.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 288969512
+| mem: 1033358})

plutus-benchmark/marlowe/test/role-payout/9.6/0403020000030204010000030001000202010101000304030001040404030100.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 264921371
+| mem: 939792})

plutus-benchmark/marlowe/test/role-payout/9.6/0405010105020401010304080005050800040301010800080207080704020206.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 295628547
+| mem: 1054612})

plutus-benchmark/marlowe/test/role-payout/9.6/041a2c3b111139201a3a2c173c392b170e16370d300f2d28342d0f2f0e182e01.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 299151875
+| mem: 1075656})

plutus-benchmark/marlowe/test/role-payout/9.6/04f592afc6e57c633b9c55246e7c82e87258f04e2fb910c37d8e2417e9db46e5.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 358290925
+| mem: 1304216})

plutus-benchmark/marlowe/test/role-payout/9.6/057ebc80922f16a5f4bf13e985bf586b8cff37a2f6fe0f3ce842178c16981027.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 244136016
+| mem: 859530})

plutus-benchmark/marlowe/test/role-payout/9.6/06317060a8e488b1219c9dae427f9ce27918a9e09ee8ac424afa33ca923f7954.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 265031754
+| mem: 936594})

plutus-benchmark/marlowe/test/role-payout/9.6/07658a6c898ad6d624c37df1e49e909c2e9349ba7f4c0a6be5f166fe239bfcae.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 233610581
+| mem: 809684})

plutus-benchmark/marlowe/test/role-payout/9.6/0bdca1cb8fa7e38e09062557b82490714052e84e2054e913092cd84ac071b961.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 297962489
+| mem: 1077996})

plutus-benchmark/marlowe/test/role-payout/9.6/0c9d3634aeae7038f839a1262d1a8bc724dc77af9426459417a56ec73240f0e0.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 259590721
+| mem: 921136})

plutus-benchmark/marlowe/test/role-payout/9.6/0d0f01050a0a0a0b0b050d0404090e0d0506000d0a041003040e0f100e0a0408.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 264572163
+| mem: 951632})

plutus-benchmark/marlowe/test/role-payout/9.6/0dbb692d2bf22d25eeceac461cfebf616f54003077a8473abc0457f18e025960.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 306383189
+| mem: 1108768})

plutus-benchmark/marlowe/test/role-payout/9.6/0e00171d0f1e1f14070d0a00091f07101808021d081e1b120219081312081e15.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 250579289
+| mem: 887724})

plutus-benchmark/marlowe/test/role-payout/9.6/0e72f62b0f922e31a2340baccc768104025400cf7fdd7dae62fbba5fc770936d.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 280611578
+| mem: 999006})

plutus-benchmark/marlowe/test/role-payout/9.6/0e97c9d9417354d9460f2eb35018d3904b7b035af16ab299258adab93be0911a.eval.golden

@@ -0,0 +1,2 @@
+({cpu: 275022771
+| mem: 977108})