wip

bench/Base.hs

@@ -36,6 +36,7 @@ data Lang
   | Juvix
   | Runtime
   | Core
+  deriving stock (Eq)
 
 instance Show Lang where
   show = \case

bench/Main.hs

@@ -1,8 +1,123 @@
 module Main where
 
-import Compile qualified
-import Juvix.Prelude
+import Base
+import Data.Text qualified as Text
+import Development.Shake hiding ((<//>))
+import Gauge
+import Juvix.Prelude.Base
+import Juvix.Prelude.Path as Path hiding (doesFileExist, (-<.>))
+import Juvix.Prelude.Path qualified as Path
+import Suites
 
 main :: IO ()
-main = do
-  Compile.compile
+main = shakeArgs opts compileRules
+  where
+    opts :: ShakeOptions
+    opts = shakeOptions
+
+compileRules :: Rules ()
+compileRules = do
+  phony "clean" $ do
+    putInfo ("Deleting " <> toFilePath resultsDir)
+    removePathForcibly resultsDir
+  forM_ suites suiteRules
+
+suiteRules :: Suite -> Rules ()
+suiteRules s = do
+  forM_ (s ^. suiteVariants) (variantRules s)
+  csvRules s
+  plotRules s
+
+multiRecipe :: [Path Abs File] -> Action () -> Rules ()
+multiRecipe out howto = map toFilePath out &%> const howto
+
+recipe :: Path Abs File -> Action () -> Rules ()
+recipe out howto = toFilePath out %> const howto
+
+variantRules :: Suite -> Variant -> Rules ()
+variantRules s v = do
+  action $ do
+    whenM
+      (doesFileExist (toFilePath srcFile))
+      (need [toFilePath exeFile])
+
+  recipe exeFile $ do
+    need [toFilePath srcFile]
+    ensureDir outDir
+    (v ^. variantBuild) args
+  where
+    args :: BuildArgs
+    args =
+      BuildArgs
+        { _buildSrc = srcFile,
+          _buildOutDir = outDir
+        }
+    lang :: Lang
+    lang = v ^. variantLanguage
+    srcFile :: Path Abs File
+    srcFile =
+      addExtension'
+        (langExtension lang)
+        (suiteSrcDir s <//> langPath lang <//> suiteBaseFile s)
+    exeFile :: Path Abs File
+    exeFile = outDir <//> replaceExtensions' (v ^. variantExtensions) (filename srcFile)
+    outDir :: Path Abs Dir
+    outDir = variantBinDir s v
+
+plotRules :: Suite -> Rules ()
+plotRules s = do
+  let pdf :: Path Abs File = suitePdfFile s
+      csv :: Path Abs File = suiteCsvFile s
+      svg :: Path Abs File = suiteSvgFile s
+      out :: Path Abs File = suitePlotFile s
+  want [toFilePath pdf, toFilePath svg]
+  multiRecipe [pdf, svg] $ do
+    need [toFilePath csv, toFilePath gnuplotFile]
+    ensureDir (parent pdf)
+    command_
+      []
+      "gnuplot"
+      ( gpArg "name" (s ^. suiteTitle)
+          ++ gpArg "outfile" (toFilePath out)
+          ++ gpArg "csvfile" (toFilePath csv)
+          ++ [toFilePath gnuplotFile]
+      )
+  where
+    gpArg :: String -> String -> [String]
+    gpArg arg val = ["-e", arg <> "='" <> val <> "'"]
+
+csvRules :: Suite -> Rules ()
+csvRules s =
+  recipe csv $ do
+    need [toFilePath (variantBinFile s v) | v <- s ^. suiteVariants]
+    ensureDir (parent csv)
+    whenM (Path.doesFileExist csv) (removeFile csv)
+    liftIO (runMode DefaultMode (config s) [] (fromSuite s) >> addColorColumn)
+  where
+    csv :: Path Abs File = suiteCsvFile s
+    addColorColumn :: IO ()
+    addColorColumn = do
+      header :| rows <- nonEmpty' . Text.lines <$> readFile (toFilePath csv)
+      let rows' =
+            [ showColour (v ^. variantColor) <> "," <> r
+              | (v, r) <- zipExact (s ^. suiteVariants) rows
+            ]
+          header' = "Color," <> header
+      writeFile (toFilePath csv) (Text.unlines (header' : rows'))
+
+fromSuite :: Suite -> [Benchmark]
+fromSuite s = map go (s ^. suiteVariants)
+  where
+    go :: Variant -> Benchmark
+    go v = bench title (nfIO ((v ^. variantRun) (variantBinFile s v)))
+      where
+        title :: String
+        title = show (v ^. variantLanguage) <> maybe "" (" " <>) (v ^. variantTitle)
+
+config :: Suite -> Config
+config s =
+  defaultConfig
+    { quickMode = False,
+      csvFile = Just (toFilePath (suiteCsvFile s)),
+      timeLimit = Just 30
+    }

bench/Suites.hs

@@ -10,15 +10,17 @@ suites =
     defaultSuite
     [ "mergesort",
       "fibonacci",
-      "ackermann", -- juvix crashes
-      "combinations", -- juvix crashes: out of call stack
-      "cps",
-      "fold",
-      "mapfold",
-      "mapfun",
-      "maybe",
-      "prime"
+      "maybe"
     ]
+    <> [ Suite "fold" (allVariantsExcept [C] [CoreEval]),
+         Suite "mapfold" (allVariantsExcept [C] [CoreEval]),
+         Suite "mapfun" (allVariantsExcept [C] [CoreEval])
+       ]
+
+-- "ackermann", -- juvix crashes
+-- "combinations", -- juvix crashes:  call stack exhausted
+-- "cps", call stack exhausted
+-- "prime" -- Address boundary error
 
 defaultSuite :: String -> Suite
 defaultSuite title =

bench/Variants.hs

@@ -8,11 +8,15 @@ import Juvix.Prelude
 import Juvix.Prelude.Env
 import System.Process
 
+allVariantsExcept :: [Lang] -> [VariantId] -> [Variant]
+allVariantsExcept ls vs = filter (\v -> (v ^. variantLanguage) `notElem` ls)
+   (map getVariant (filter (`notElem` vs) allElements))
+
 allVariants :: [Variant]
 allVariants = map getVariant allElements
 
 defaultVariants :: [Variant]
-defaultVariants = map getVariant (delete CoreEval allElements)
+defaultVariants = allVariantsExcept [] [CoreEval]
 
 data VariantId
   = OcamlExe

gnuplot/bars.gp

@@ -19,6 +19,7 @@ set ylabel "execution time (s)" font ", 20"
 set style fill solid
 unset key
 set yrange [0 : *]
+set offsets graph 0,0.5
 
 set multiplot layout 2, 1 title ('suite '.name) font ",24" scale 1, 1