From 7d2a59cc9ff82724ec0efe01f91c4a8fc5d25d2a Mon Sep 17 00:00:00 2001 From: Jan Mas Rovira Date: Mon, 15 Jul 2024 10:02:48 +0200 Subject: [PATCH] Add precondition to run tests (#2887) Some tests require external dependencies, such as `rustc`, `wasmer`, `run_cairo_vm.sh`, etc. If one does not have some of these available on their computer, then the test suite will have a lot of failed tests with the same fail message `X is not on $PATH`. This can be a bit ditracting and it slows running the test suite. I've introduced some preconditions that are checked before the actual test suite so that if some of these commands are not on path then the tests that need them are not run. Instead, you get a single failed test (for each of the subtrees that failed the precondition). --- test/Base.hs | 6 +++ test/Casm.hs | 10 ++++- test/Casm/Compilation.hs | 14 +++++-- test/Casm/Compilation/Base.hs | 6 ++- test/Casm/Compilation/Positive.hs | 25 +++++++----- test/Casm/Reg.hs | 14 +++++-- test/Casm/Reg/Base.hs | 4 +- test/Casm/Reg/Cairo.hs | 42 ++++++++++---------- test/Casm/Run.hs | 8 ++-- test/Casm/Run/Base.hs | 38 +++++++++++------- test/Casm/Run/Positive.hs | 20 ++++++---- test/Main.hs | 64 ++++++++++++++++--------------- test/Rust.hs | 6 ++- test/Rust/Compilation.hs | 7 +++- test/Rust/Compilation/Base.hs | 7 ++-- test/Rust/RiscZero.hs | 7 +++- test/Rust/RiscZero/Base.hs | 11 +++--- test/VampIR.hs | 19 ++++++--- test/VampIR/Core/Base.hs | 7 ++-- 19 files changed, 194 insertions(+), 121 deletions(-) diff --git a/test/Base.hs b/test/Base.hs index 8549e6f7f1..4d961d04ec 100644 --- a/test/Base.hs +++ b/test/Base.hs @@ -10,6 +10,7 @@ module Base ) where +import Control.Exception qualified as E import Control.Monad.Extra as Monad import Data.Algorithm.Diff import Data.Algorithm.DiffOutput @@ -60,6 +61,11 @@ mkTest TestDescr {..} = case _testAssertion of Single assertion -> testCase _testName (withCurrentDir _testRoot assertion) Steps steps -> testCaseSteps _testName (withCurrentDir _testRoot . steps) +withPrecondition :: Assertion -> IO TestTree -> IO TestTree +withPrecondition assertion ifSuccess = do + E.catch (assertion >> ifSuccess) $ \case + E.SomeException e -> return (testCase "Precondition failed" (assertFailure (show e))) + assertEqDiffText :: String -> Text -> Text -> Assertion assertEqDiffText = assertEqDiff unpack diff --git a/test/Casm.hs b/test/Casm.hs index fb4d7c6e15..5825a14dcf 100644 --- a/test/Casm.hs +++ b/test/Casm.hs @@ -5,5 +5,11 @@ import Casm.Compilation qualified as Compile import Casm.Reg qualified as Reg import Casm.Run qualified as Run -allTests :: TestTree -allTests = testGroup "CASM tests" [Run.allTests, Reg.allTests, Compile.allTests] +allTests :: IO TestTree +allTests = + testGroup "CASM tests" + <$> sequence + [ Run.allTests, + Reg.allTests, + Compile.allTests + ] diff --git a/test/Casm/Compilation.hs b/test/Casm/Compilation.hs index 1c7d3b6fd8..3b94d1bcbe 100644 --- a/test/Casm/Compilation.hs +++ b/test/Casm/Compilation.hs @@ -1,8 +1,14 @@ module Casm.Compilation where import Base -import Casm.Compilation.Negative qualified as N -import Casm.Compilation.Positive qualified as P +import Casm.Compilation.Negative qualified as Negative +import Casm.Compilation.Positive qualified as Positive -allTests :: TestTree -allTests = testGroup "Juvix to CASM compilation" [P.allTests, P.allTestsNoOptimize, N.allTests] +allTests :: IO TestTree +allTests = + testGroup "Juvix to CASM compilation" + <$> sequence + [ Positive.allTests, + Positive.allTestsNoOptimize, + return Negative.allTests + ] diff --git a/test/Casm/Compilation/Base.hs b/test/Casm/Compilation/Base.hs index 15ab10f8a6..bfb01975a5 100644 --- a/test/Casm/Compilation/Base.hs +++ b/test/Casm/Compilation/Base.hs @@ -1,4 +1,8 @@ -module Casm.Compilation.Base where +module Casm.Compilation.Base + ( module Casm.Compilation.Base, + cairoVmPrecondition, + ) +where import Base import Casm.Run.Base diff --git a/test/Casm/Compilation/Positive.hs b/test/Casm/Compilation/Positive.hs index d80071050f..ee191c9ad1 100644 --- a/test/Casm/Compilation/Positive.hs +++ b/test/Casm/Compilation/Positive.hs @@ -31,17 +31,22 @@ toTestDescr optLevel PosTest {..} = _testAssertion = Steps $ compileAssertion _dir _interp _runVM optLevel file' input' expected' } -allTests :: TestTree -allTests = - testGroup - "Juvix to CASM positive tests" - (map (mkTest . toTestDescr 3) tests) +mkAllTests :: String -> Int -> IO TestTree +mkAllTests title optimLevel = do + let (vmTests, nonVmTests) = partition (^. runVM) tests + vmGroup = testGroup "With VM" (mkTest . toTestDescr optimLevel <$> vmTests) + vmTestTree <- withPrecondition cairoVmPrecondition (return vmGroup) + let nonVmTestTree = testGroup "Without VM" (mkTest . toTestDescr optimLevel <$> nonVmTests) + return $ + testGroup + title + [vmTestTree, nonVmTestTree] -allTestsNoOptimize :: TestTree -allTestsNoOptimize = - testGroup - "Juvix to CASM positive tests (no optimization)" - (map (mkTest . toTestDescr 0) tests) +allTests :: IO TestTree +allTests = mkAllTests "CASM run positive tests" 3 + +allTestsNoOptimize :: IO TestTree +allTestsNoOptimize = mkAllTests "Juvix to CASM positive tests (no optimization)" 0 posTest :: String -> Bool -> Bool -> Path Rel Dir -> Path Rel File -> Maybe (Path Rel File) -> Path Rel File -> PosTest posTest _name _interp _runVM rdir rfile rinfile routfile = diff --git a/test/Casm/Reg.hs b/test/Casm/Reg.hs index e1ea14b92c..8bbe63f7f9 100644 --- a/test/Casm/Reg.hs +++ b/test/Casm/Reg.hs @@ -1,8 +1,14 @@ module Casm.Reg where import Base -import Casm.Reg.Cairo qualified as C -import Casm.Reg.Positive qualified as P +import Casm.Reg.Cairo qualified as Cairo +import Casm.Reg.Positive qualified as Positive -allTests :: TestTree -allTests = testGroup "JuvixReg to CASM translation" [P.allTests, C.allTests] +allTests :: IO TestTree +allTests = + testGroup + "JuvixReg to CASM translation" + <$> sequence + [ return Positive.allTests, + Cairo.allTests + ] diff --git a/test/Casm/Reg/Base.hs b/test/Casm/Reg/Base.hs index 245e73547b..801db6d8ee 100644 --- a/test/Casm/Reg/Base.hs +++ b/test/Casm/Reg/Base.hs @@ -13,7 +13,7 @@ import Reg.Run.Base qualified as Reg compileAssertion' :: EntryPoint -> Maybe (Path Abs File) -> Path Abs Dir -> Path Abs File -> Symbol -> Reg.InfoTable -> (String -> IO ()) -> Assertion compileAssertion' entryPoint inputFile _ outputFile _ tab step = do step "Translate to CASM" - case run $ runError @JuvixError $ runReader entryPoint $ regToCasm tab of + case run . runError @JuvixError . runReader entryPoint $ regToCasm tab of Left err -> assertFailure (prettyString (fromJuvixError @GenericError err)) Right Result {..} -> do step "Interpret" @@ -30,7 +30,7 @@ compileAssertion' entryPoint inputFile _ outputFile _ tab step = do cairoAssertion' :: EntryPoint -> Maybe (Path Abs File) -> Path Abs Dir -> Path Abs File -> Symbol -> Reg.InfoTable -> (String -> IO ()) -> Assertion cairoAssertion' entryPoint inputFile dirPath outputFile _ tab step = do step "Translate to Cairo" - case run $ runError @JuvixError $ runReader entryPoint $ regToCairo tab of + case run . runError @JuvixError . runReader entryPoint $ regToCairo tab of Left err -> assertFailure (prettyString (fromJuvixError @GenericError err)) Right res -> do step "Serialize to Cairo bytecode" diff --git a/test/Casm/Reg/Cairo.hs b/test/Casm/Reg/Cairo.hs index 6cff39dcb7..05b541464a 100644 --- a/test/Casm/Reg/Cairo.hs +++ b/test/Casm/Reg/Cairo.hs @@ -3,6 +3,7 @@ module Casm.Reg.Cairo where import Base import Casm.Reg.Base import Casm.Reg.Positive qualified as P +import Casm.Run.Base (cairoVmPrecondition) testDescr :: P.PosTest -> TestDescr testDescr P.PosTest {..} = @@ -16,27 +17,28 @@ testDescr P.PosTest {..} = _testAssertion = Steps $ regToCairoAssertion tRoot file' input' expected' } -allTests :: TestTree +allTests :: IO TestTree allTests = - testGroup - "JuvixReg to Cairo translation positive tests" - ( map (mkTest . testDescr) $ - P.filterOutTests - [ "Test001: Arithmetic opcodes", - "Test013: Fibonacci numbers in linear time", - "Test014: Trees", - "Test016: Arithmetic", - "Test017: Closures as arguments", - "Test023: McCarthy's 91 function", - "Test024: Higher-order recursive functions", - "Test027: Fast exponentiation", - "Test030: Mutual recursion", - "Test031: Temporary stack with branching", - "Test036: Streams without memoization" - ] - P.tests - ++ cairoTests - ) + withPrecondition cairoVmPrecondition + . return + . testGroup + "JuvixReg to Cairo translation positive tests" + $ map (mkTest . testDescr) + $ P.filterOutTests + [ "Test001: Arithmetic opcodes", + "Test013: Fibonacci numbers in linear time", + "Test014: Trees", + "Test016: Arithmetic", + "Test017: Closures as arguments", + "Test023: McCarthy's 91 function", + "Test024: Higher-order recursive functions", + "Test027: Fast exponentiation", + "Test030: Mutual recursion", + "Test031: Temporary stack with branching", + "Test036: Streams without memoization" + ] + P.tests + ++ cairoTests cairoTests :: [P.PosTest] cairoTests = diff --git a/test/Casm/Run.hs b/test/Casm/Run.hs index 102ffbd47a..f9ea361709 100644 --- a/test/Casm/Run.hs +++ b/test/Casm/Run.hs @@ -1,8 +1,8 @@ module Casm.Run where import Base -import Casm.Run.Negative qualified as RunN -import Casm.Run.Positive qualified as RunP +import Casm.Run.Negative qualified as Negative +import Casm.Run.Positive qualified as Positive -allTests :: TestTree -allTests = testGroup "CASM run" [RunP.allTests, RunN.allTests] +allTests :: IO TestTree +allTests = testGroup "CASM run" <$> sequence [Positive.allTests, return Negative.allTests] diff --git a/test/Casm/Run/Base.hs b/test/Casm/Run/Base.hs index 2976b7b3ce..3bb3faa3d6 100644 --- a/test/Casm/Run/Base.hs +++ b/test/Casm/Run/Base.hs @@ -18,24 +18,25 @@ casmRunVM' dirPath outputFile inputFile = do let args = maybe [] (\f -> ["--program_input", toFilePath f]) inputFile readProcessCwd (toFilePath dirPath) "run_cairo_vm.sh" (toFilePath outputFile : args) "" +cairoVmPrecondition :: Assertion +cairoVmPrecondition = do + assertCmdExists $(mkRelFile "run_cairo_vm.sh") + casmRunVM :: EntryPoint -> LabelInfo -> Code -> [Builtin] -> Int -> Maybe (Path Abs File) -> Path Abs File -> (String -> IO ()) -> Assertion casmRunVM entryPoint labi instrs blts outputSize inputFile expectedFile step = do - step "Check run_cairo_vm.sh is on path" - assertCmdExists $(mkRelFile "run_cairo_vm.sh") withTempDir' ( \dirPath -> do step "Serialize to Cairo bytecode" let res = - run $ - runReader entryPoint $ - casmToCairo - ( Casm.Result - { _resultLabelInfo = labi, - _resultCode = instrs, - _resultBuiltins = blts, - _resultOutputSize = outputSize - } - ) + run + . runReader entryPoint + $ casmToCairo + Casm.Result + { _resultLabelInfo = labi, + _resultCode = instrs, + _resultBuiltins = blts, + _resultOutputSize = outputSize + } outputFile = dirPath $(mkRelFile "out.json") encodeFile (toFilePath outputFile) res step "Run Cairo VM" @@ -66,7 +67,18 @@ casmInterpret labi instrs inputFile expectedFile step = assertEqDiffText ("Check: RUN output = " <> toFilePath expectedFile) actualOutput expected ) -casmRunAssertion' :: EntryPoint -> Bool -> Bool -> LabelInfo -> Code -> [Builtin] -> Int -> Maybe (Path Abs File) -> Path Abs File -> (String -> IO ()) -> Assertion +casmRunAssertion' :: + EntryPoint -> + Bool -> + Bool -> + LabelInfo -> + Code -> + [Builtin] -> + Int -> + Maybe (Path Abs File) -> + Path Abs File -> + (String -> IO ()) -> + Assertion casmRunAssertion' entryPoint bInterp bRunVM labi instrs blts outputSize inputFile expectedFile step = case validate labi instrs of Left err -> do diff --git a/test/Casm/Run/Positive.hs b/test/Casm/Run/Positive.hs index 2e4acf4bc2..20d42d9ef2 100644 --- a/test/Casm/Run/Positive.hs +++ b/test/Casm/Run/Positive.hs @@ -13,6 +13,8 @@ data PosTest = PosTest _inputFile :: Maybe (Path Rel File) } +makeLenses ''PosTest + root :: Path Abs Dir root = relToProject $(mkRelDir "tests/Casm/positive") @@ -28,14 +30,16 @@ testDescr PosTest {..} = _testAssertion = Steps $ casmRunAssertion _interp _runVM tRoot file' input' expected' } -filterTests :: [String] -> [PosTest] -> [PosTest] -filterTests incl = filter (\PosTest {..} -> _name `elem` incl) - -allTests :: TestTree -allTests = - testGroup - "CASM run positive tests" - (map (mkTest . testDescr) tests) +allTests :: IO TestTree +allTests = do + let (vmTests, nonVmTests) = partition (^. runVM) tests + vmGroup = testGroup "With VM" (mkTest . testDescr <$> vmTests) + vmTestTree <- withPrecondition cairoVmPrecondition (return vmGroup) + let nonVmTestTree = testGroup "Without VM" (mkTest . testDescr <$> nonVmTests) + return $ + testGroup + "CASM run positive tests" + [vmTestTree, nonVmTestTree] tests :: [PosTest] tests = diff --git a/test/Main.hs b/test/Main.hs index 6d95b71ee6..1a7d39e9b9 100644 --- a/test/Main.hs +++ b/test/Main.hs @@ -25,41 +25,45 @@ import Tree qualified import Typecheck qualified import VampIR qualified -slowTests :: TestTree +slowTests :: IO TestTree slowTests = sequentialTestGroup "Juvix slow tests" AllFinish - [ Runtime.allTests, - Reg.allTests, - Asm.allTests, - Tree.allTests, - Core.allTests, - Internal.allTests, - Compilation.allTests, - Examples.allTests, - Rust.allTests, - Casm.allTests, - VampIR.allTests, - Anoma.allTests, - Repl.allTests - ] + <$> sequence + [ return Runtime.allTests, + return Reg.allTests, + return Asm.allTests, + return Tree.allTests, + return Core.allTests, + return Internal.allTests, + return Compilation.allTests, + return Examples.allTests, + Rust.allTests, + Casm.allTests, + VampIR.allTests, + return Anoma.allTests, + return Repl.allTests + ] -fastTests :: TestTree +fastTests :: IO TestTree fastTests = - testGroup - "Juvix fast tests" - [ Parsing.allTests, - Resolver.allTests, - Scope.allTests, - Termination.allTests, - Typecheck.allTests, - Format.allTests, - Formatter.allTests, - Package.allTests, - BackendMarkdown.allTests, - Nockma.allTests - ] + return $ + testGroup + "Juvix fast tests" + [ Parsing.allTests, + Resolver.allTests, + Scope.allTests, + Termination.allTests, + Typecheck.allTests, + Format.allTests, + Formatter.allTests, + Package.allTests, + BackendMarkdown.allTests, + Nockma.allTests + ] main :: IO () -main = defaultMain (testGroup "Juvix tests" [fastTests, slowTests]) +main = do + tests <- sequence [fastTests, slowTests] + defaultMain (testGroup "Juvix tests" tests) diff --git a/test/Rust.hs b/test/Rust.hs index 6aedb10de3..18c9f09193 100644 --- a/test/Rust.hs +++ b/test/Rust.hs @@ -4,5 +4,7 @@ import Base import Rust.Compilation qualified as Compilation import Rust.RiscZero qualified as RiscZero -allTests :: TestTree -allTests = sequentialTestGroup "Juvix to Rust tests" AllFinish [Compilation.allTests, RiscZero.allTests] +allTests :: IO TestTree +allTests = + sequentialTestGroup "Juvix to Rust tests" AllFinish + <$> sequence [Compilation.allTests, RiscZero.allTests] diff --git a/test/Rust/Compilation.hs b/test/Rust/Compilation.hs index 59064cba33..7d62955332 100644 --- a/test/Rust/Compilation.hs +++ b/test/Rust/Compilation.hs @@ -1,7 +1,10 @@ module Rust.Compilation where import Base +import Rust.Compilation.Base import Rust.Compilation.Positive qualified as P -allTests :: TestTree -allTests = testGroup "Juvix to native Rust compilation tests" [P.allTests, P.allTestsNoOptimize] +allTests :: IO TestTree +allTests = + withPrecondition precondition . return $ + testGroup "Juvix to native Rust compilation tests" [P.allTests, P.allTestsNoOptimize] diff --git a/test/Rust/Compilation/Base.hs b/test/Rust/Compilation/Base.hs index bf9ab868f7..c2d2db2b30 100644 --- a/test/Rust/Compilation/Base.hs +++ b/test/Rust/Compilation/Base.hs @@ -7,6 +7,10 @@ import Juvix.Compiler.Backend.Rust.Pretty import Juvix.Compiler.Core qualified as Core import System.Process qualified as P +precondition :: Assertion +precondition = do + assertCmdExists $(mkRelFile "rustc") + compileAssertion :: Path Abs Dir -> Int -> @@ -27,9 +31,6 @@ compileAssertion root' optLevel mainFile expectedFile step = do let inputFile = dirPath $(mkRelFile "Program.rs") writeFileEnsureLn inputFile _resultRustCode - step "Check rustc is on path" - assertCmdExists $(mkRelFile "rustc") - expected <- readFile expectedFile let executeNative :: Path Abs File -> IO Text diff --git a/test/Rust/RiscZero.hs b/test/Rust/RiscZero.hs index 25c9a36933..b36679615d 100644 --- a/test/Rust/RiscZero.hs +++ b/test/Rust/RiscZero.hs @@ -1,7 +1,10 @@ module Rust.RiscZero where import Base +import Rust.RiscZero.Base import Rust.RiscZero.Positive qualified as P -allTests :: TestTree -allTests = sequentialTestGroup "Juvix to RISC0 Rust compilation tests" AllFinish [P.allTests, P.allTestsNoOptimize] +allTests :: IO TestTree +allTests = + withPrecondition precondition . return $ + sequentialTestGroup "Juvix to RISC0 Rust compilation tests" AllFinish [P.allTests, P.allTestsNoOptimize] diff --git a/test/Rust/RiscZero/Base.hs b/test/Rust/RiscZero/Base.hs index 1212e0cb3a..a5a16bf3b7 100644 --- a/test/Rust/RiscZero/Base.hs +++ b/test/Rust/RiscZero/Base.hs @@ -10,6 +10,11 @@ import System.Directory import System.Environment import System.Process qualified as P +precondition :: Assertion +precondition = do + assertCmdExists $(mkRelFile "cargo") + assertCmdExists $(mkRelFile "r0vm") + compileAssertion :: IO (Path Abs Dir) -> Path Abs Dir -> @@ -41,12 +46,6 @@ compileAssertion tmpDir' root' optLevel mainFile expectedFile step = do $(mkRelFile "main.rs") writeFileEnsureLn outFile _resultRustCode - step "Check cargo is on path" - assertCmdExists $(mkRelFile "cargo") - - step "Check r0vm is on path" - assertCmdExists $(mkRelFile "r0vm") - expected <- readFile expectedFile step "Compile Rust to RISC0" diff --git a/test/VampIR.hs b/test/VampIR.hs index d43ce75e99..826d584304 100644 --- a/test/VampIR.hs +++ b/test/VampIR.hs @@ -1,9 +1,18 @@ module VampIR where import Base -import VampIR.Compilation.Negative qualified as N -import VampIR.Compilation.Positive qualified as PC -import VampIR.Core.Positive qualified as PT +import VampIR.Compilation.Negative qualified as Negative +import VampIR.Compilation.Positive qualified as CompilationPositive +import VampIR.Core.Base +import VampIR.Core.Positive qualified as CorePositive -allTests :: TestTree -allTests = testGroup "VampIR tests" [PT.allTests, PC.allTests, N.allTests] +allTests :: IO TestTree +allTests = + withPrecondition precondition + . return + $ testGroup + "VampIR tests" + [ CorePositive.allTests, + CompilationPositive.allTests, + Negative.allTests + ] diff --git a/test/VampIR/Core/Base.hs b/test/VampIR/Core/Base.hs index a37d5ebc88..afb73b424a 100644 --- a/test/VampIR/Core/Base.hs +++ b/test/VampIR/Core/Base.hs @@ -18,6 +18,10 @@ vampirAssertion backend root mainFile dataFile step = do entryPoint <- testDefaultEntryPointIO root mainFile vampirAssertion' entryPoint backend tab dataFile step +precondition :: Assertion +precondition = do + assertCmdExists $(mkRelFile "vamp-ir") + vampirAssertion' :: EntryPoint -> VampirBackend -> InfoTable -> Path Abs File -> (String -> IO ()) -> Assertion vampirAssertion' entryPoint backend tab dataFile step = do withTempDir' @@ -29,9 +33,6 @@ vampirAssertion' entryPoint backend tab dataFile step = do Right VampIR.Result {..} -> do writeFileEnsureLn vampirFile _resultCode - step "Check vamp-ir on path" - assertCmdExists $(mkRelFile "vamp-ir") - vampirSetupArgs backend dirPath step step "VampIR compile"