New compilation pipeline (#1832)

* Depends on PR #1824 * Closes #1556 * Closes #1825 * Closes #1843 * Closes #1729 * Closes #1596 * Closes #1343 * Closes #1382 * Closes #1867 * Closes #1876 * Changes the `juvix compile` command to use the new pipeline. * Removes the `juvix dev minic` command and the `BackendC` tests. * Adds the `juvix eval` command. * Fixes bugs in the Nat-to-integer conversion. * Fixes bugs in the Internal-to-Core and Core-to-Core.Stripped translations. * Fixes bugs in the RemoveTypeArgs transformation. * Fixes bugs in lambda-lifting (incorrect de Bruijn indices in the types of added binders). * Fixes several other bugs in the compilation pipeline. * Adds a separate EtaExpandApps transformation to avoid quadratic runtime in the Internal-to-Core translation due to repeated calls to etaExpandApps. * Changes Internal-to-Core to avoid generating matches on values which don't have an inductive type. --------- Co-authored-by: Paul Cadman <git@paulcadman.dev> Co-authored-by: janmasrovira <janmasrovira@gmail.com>
anoma · Mar 14, 2023 · 2d798ec · 2d798ec
1 parent 0f29b3e
commit 2d798ec
Show file tree

Hide file tree

Showing 121 changed files with 2,078 additions and 1,609 deletions.
diff --git a/Makefile b/Makefile
@@ -264,7 +264,7 @@ SMOKE := $(shell command -v smoke 2> /dev/null)
 
 .PHONY : smoke-only
 smoke-only:
-	@$(if $(SMOKE),, $(error "Smoke not found, please install it from https://github.com/SamirTalwar/smoke"))
+	@$(if $(SMOKE),, $(error "Smoke not found, please install it from https://github.com/jonaprieto/smoke"))
 	@smoke $(shell find tests -name '*.smoke.yaml')
 
 .PHONY : smoke

diff --git a/app/Commands/Compile.hs b/app/Commands/Compile.hs
@@ -2,191 +2,30 @@ module Commands.Compile where
 
 import Commands.Base
 import Commands.Compile.Options
-import Data.ByteString qualified as BS
-import Data.FileEmbed qualified as FE
+import Commands.Dev.Core.Compile.Base qualified as Compile
+import Commands.Extra.Compile qualified as Compile
 import Data.Text.IO qualified as TIO
-import Juvix.Compiler.Backend.C.Translation.FromInternal qualified as MiniC
-import System.Environment
-import System.Process qualified as P
+import Juvix.Compiler.Core qualified as Core
+import Juvix.Compiler.Core.Pretty qualified as Core
 
 runCommand :: (Members '[Embed IO, App] r) => CompileOptions -> Sem r ()
 runCommand opts@CompileOptions {..} = do
-  miniC <- (^. MiniC.resultCCode) <$> runPipeline _compileInputFile upToMiniC
   inputFile <- someBaseToAbs' (_compileInputFile ^. pathPath)
-  result <- runCompile inputFile opts miniC
-  case result of
-    Left err -> printFailureExit err
-    _ -> return ()
-
-inputCFile :: (Members '[App] r) => Path Abs File -> Sem r (Path Abs File)
-inputCFile inputFileCompile = do
-  buildDir <- askBuildDir
-  return (buildDir <//> outputMiniCFile)
-  where
-    outputMiniCFile :: Path Rel File
-    outputMiniCFile = replaceExtension' ".c" (filename inputFileCompile)
-
-runCompile :: (Members '[Embed IO, App] r) => Path Abs File -> CompileOptions -> Text -> Sem r (Either Text ())
-runCompile inputFileCompile o minic = do
-  buildDir <- askBuildDir
-  ensureDir buildDir
-  f <- inputCFile inputFileCompile
-  embed (TIO.writeFile (toFilePath f) minic)
-  prepareRuntime o
-  case o ^. compileTarget of
-    TargetWasm -> runError (clangCompile inputFileCompile o)
-    TargetC -> return (Right ())
-    TargetNative -> runError (clangNativeCompile inputFileCompile o)
-
-prepareRuntime :: forall r. (Members '[Embed IO, App] r) => CompileOptions -> Sem r ()
-prepareRuntime o = mapM_ writeRuntime runtimeProjectDir
-  where
-    runtimeProjectDir :: [(Path Rel File, BS.ByteString)]
-    runtimeProjectDir = case o ^. compileTarget of
-      TargetNative -> libcRuntime
-      _ -> case o ^. compileRuntime of
-        RuntimeWasiStandalone -> wasiStandaloneRuntimeDir <> builtinCRuntimeDir <> wallocDir
-        RuntimeWasiLibC -> libcRuntime
-        RuntimeStandalone -> standaloneRuntimeDir <> builtinCRuntimeDir <> wallocDir
-
-    writeRuntime :: (Path Rel File, BS.ByteString) -> Sem r ()
-    writeRuntime (filePath, contents) = do
-      buildDir <- askBuildDir
-      embed (BS.writeFile (toFilePath (buildDir <//> filePath)) contents)
-
-wasiStandaloneRuntimeDir :: [(Path Rel File, BS.ByteString)]
-wasiStandaloneRuntimeDir = map (first relFile) $(FE.makeRelativeToProject "c-runtime/wasi-standalone" >>= FE.embedDir)
-
-standaloneRuntimeDir :: [(Path Rel File, BS.ByteString)]
-standaloneRuntimeDir = map (first relFile) $(FE.makeRelativeToProject "c-runtime/standalone" >>= FE.embedDir)
-
-wasiLibCRuntimeDir :: [(Path Rel File, BS.ByteString)]
-wasiLibCRuntimeDir = map (first relFile) $(FE.makeRelativeToProject "c-runtime/wasi-libc" >>= FE.embedDir)
-
-builtinCRuntimeDir :: [(Path Rel File, BS.ByteString)]
-builtinCRuntimeDir = map (first relFile) $(FE.makeRelativeToProject "c-runtime/builtins" >>= FE.embedDir)
-
-wallocDir :: [(Path Rel File, BS.ByteString)]
-wallocDir = map (first relFile) $(FE.makeRelativeToProject "c-runtime/walloc" >>= FE.embedDir)
-
-libcRuntime :: [(Path Rel File, BS.ByteString)]
-libcRuntime = wasiLibCRuntimeDir <> builtinCRuntimeDir
-
-clangNativeCompile ::
-  forall r.
-  (Members '[Embed IO, App, Error Text] r) =>
-  Path Abs File ->
-  CompileOptions ->
-  Sem r ()
-clangNativeCompile inputFileCompile o = do
-  inputFile <- getInputFile
-  outputFile <- getOutputFile
-  buildDir <- askBuildDir
-  runClang (nativeArgs buildDir outputFile inputFile)
-  where
-    getOutputFile :: Sem r (Path Abs File)
-    getOutputFile = case o ^. compileOutputFile of
-      Nothing -> return (removeExtension' inputFileCompile)
-      Just f -> someBaseToAbs' (f ^. pathPath)
-
-    getInputFile :: Sem r (Path Abs File)
-    getInputFile = inputCFile inputFileCompile
-
-clangCompile ::
-  forall r.
-  (Members '[Embed IO, App, Error Text] r) =>
-  Path Abs File ->
-  CompileOptions ->
-  Sem r ()
-clangCompile inputFileCompile o = do
-  outputFile <- getOutputFile
-  inputFile <- getInputFile
-  let clangArgs :: Sem r [String]
-      clangArgs = case o ^. compileRuntime of
-        RuntimeStandalone -> do
-          standaloneLibArgs outputFile inputFile
-        RuntimeWasiStandalone -> wasiStandaloneArgs outputFile inputFile
-        RuntimeWasiLibC -> wasiLibcArgs outputFile inputFile
-
-  clangArgs >>= runClang
-  where
-    getOutputFile :: Sem r (Path Abs File)
-    getOutputFile = maybe (return defaultOutputFile) someBaseToAbs' (o ^? compileOutputFile . _Just . pathPath)
-
-    defaultOutputFile :: Path Abs File
-    defaultOutputFile = replaceExtension' ".wasm" inputFileCompile
-
-    getInputFile :: Sem r (Path Abs File)
-    getInputFile = inputCFile inputFileCompile
-
-sysrootEnvVar :: (Members '[Error Text, Embed IO] r) => Sem r (Path Abs Dir)
-sysrootEnvVar =
-  absDir
-    <$> fromMaybeM (throw msg) (embed (lookupEnv "WASI_SYSROOT_PATH"))
-  where
-    msg :: Text
-    msg = "Missing environment variable WASI_SYSROOT_PATH"
-
-commonArgs :: Path Abs Dir -> Path Abs File -> [String]
-commonArgs buildDir wasmOutputFile =
-  [ "-std=c99",
-    "-Oz",
-    "-I",
-    toFilePath buildDir,
-    "-o",
-    toFilePath wasmOutputFile
-  ]
-
-standaloneLibArgs :: (Members '[App, Embed IO] r) => Path Abs File -> Path Abs File -> Sem r [String]
-standaloneLibArgs wasmOutputFile inputFile = do
-  buildDir <- askBuildDir
-  return $
-    commonArgs buildDir wasmOutputFile
-      <> [ "--target=wasm32",
-           "-nodefaultlibs",
-           "-nostartfiles",
-           "-Wl,--no-entry",
-           toFilePath (buildDir <//> $(mkRelFile "walloc.c")),
-           toFilePath inputFile
-         ]
-
-wasiStandaloneArgs :: (Members '[App, Error Text, Embed IO] r) => Path Abs File -> Path Abs File -> Sem r [String]
-wasiStandaloneArgs wasmOutputFile inputFile = do
-  buildDir <- askBuildDir
-  com <- wasiCommonArgs wasmOutputFile
-  return $
-    com
-      <> [ toFilePath (buildDir <//> $(mkRelFile "walloc.c")),
-           toFilePath inputFile
-         ]
-
-wasiLibcArgs :: (Members '[App, Error Text, Embed IO] r) => Path Abs File -> Path Abs File -> Sem r [String]
-wasiLibcArgs wasmOutputFile inputFile = do
-  com <- wasiCommonArgs wasmOutputFile
-  return $ com <> ["-lc", toFilePath inputFile]
-
-nativeArgs :: Path Abs Dir -> Path Abs File -> Path Abs File -> [String]
-nativeArgs buildDir outputFile inputFile =
-  commonArgs buildDir outputFile <> [toFilePath inputFile]
-
-wasiCommonArgs :: (Members '[App, Error Text, Embed IO] r) => Path Abs File -> Sem r [String]
-wasiCommonArgs wasmOutputFile = do
-  sysrootPath <- sysrootEnvVar
-  buildDir <- askBuildDir
-  return $
-    commonArgs buildDir wasmOutputFile
-      <> [ "-nodefaultlibs",
-           "--target=wasm32-wasi",
-           "--sysroot",
-           toFilePath sysrootPath
-         ]
-
-runClang ::
-  (Members '[Embed IO, Error Text] r) =>
-  [String] ->
-  Sem r ()
-runClang args = do
-  (exitCode, _, err) <- embed (P.readProcessWithExitCode "clang" args "")
-  case exitCode of
-    ExitSuccess -> return ()
-    _ -> throw (pack err)
+  Core.CoreResult {..} <- runPipeline _compileInputFile upToCore
+  let arg =
+        Compile.PipelineArg
+          { _pipelineArgFile = inputFile,
+            _pipelineArgOptions = opts,
+            _pipelineArgInfoTable = _coreResultTable
+          }
+  case _compileTarget of
+    TargetNative64 -> Compile.runCPipeline arg
+    TargetWasm32Wasi -> Compile.runCPipeline arg
+    TargetGeb -> Compile.runGebPipeline arg
+    TargetCore -> writeCoreFile arg
+    TargetAsm -> Compile.runAsmPipeline arg
+
+writeCoreFile :: (Members '[Embed IO, App] r) => Compile.PipelineArg -> Sem r ()
+writeCoreFile Compile.PipelineArg {..} = do
+  coreFile <- Compile.outputFile _pipelineArgOptions _pipelineArgFile
+  embed $ TIO.writeFile (toFilePath coreFile) (show $ Core.ppOut Core.defaultOptions {Core._optShowDeBruijnIndices = True} _pipelineArgInfoTable)
diff --git a/app/Commands/Compile/Options.hs b/app/Commands/Compile/Options.hs
@@ -1,77 +1,6 @@
-module Commands.Compile.Options where
+module Commands.Compile.Options
+  ( module Commands.Extra.Compile.Options,
+  )
+where
 
-import CommonOptions
-
-data CompileTarget
-  = TargetC
-  | TargetWasm
-  | TargetNative
-  deriving stock (Show, Data)
-
-data CompileRuntime
-  = RuntimeWasiStandalone
-  | RuntimeWasiLibC
-  | RuntimeStandalone
-  deriving stock (Show, Data)
-
-data CompileOptions = CompileOptions
-  { _compileTarget :: CompileTarget,
-    _compileRuntime :: CompileRuntime,
-    _compileOutputFile :: Maybe (AppPath File),
-    _compileInputFile :: AppPath File
-  }
-  deriving stock (Data)
-
-makeLenses ''CompileOptions
-
-parseCompile :: Parser CompileOptions
-parseCompile = do
-  _compileTarget <-
-    option
-      (eitherReader parseTarget)
-      ( long "target"
-          <> short 't'
-          <> metavar "TARGET"
-          <> value TargetNative
-          <> showDefaultWith targetShow
-          <> help "select a target: wasm, c, native"
-      )
-  _compileRuntime <-
-    option
-      (eitherReader parseRuntime)
-      ( long "runtime"
-          <> short 'r'
-          <> metavar "RUNTIME"
-          <> value RuntimeWasiStandalone
-          <> showDefaultWith runtimeShow
-          <> help "select a runtime: wasi-standalone, wasi-libc, standalone"
-      )
-  _compileOutputFile <- optional parseGenericOutputFile
-  _compileInputFile <- parseInputJuvixFile
-  pure CompileOptions {..}
-  where
-    parseTarget :: String -> Either String CompileTarget
-    parseTarget = \case
-      "wasm" -> Right TargetWasm
-      "c" -> Right TargetC
-      "native" -> Right TargetNative
-      s -> Left $ "unrecognised target: " <> s
-
-    targetShow :: CompileTarget -> String
-    targetShow = \case
-      TargetC -> "c"
-      TargetWasm -> "wasm"
-      TargetNative -> "native"
-
-    parseRuntime :: String -> Either String CompileRuntime
-    parseRuntime = \case
-      "wasi-standalone" -> Right RuntimeWasiStandalone
-      "wasi-libc" -> Right RuntimeWasiLibC
-      "standalone" -> Right RuntimeStandalone
-      s -> Left $ "unrecognised runtime: " <> s
-
-    runtimeShow :: CompileRuntime -> String
-    runtimeShow = \case
-      RuntimeWasiStandalone -> "wasi-standalone"
-      RuntimeWasiLibC -> "wasi-libc"
-      RuntimeStandalone -> "standalone"
+import Commands.Extra.Compile.Options
diff --git a/app/Commands/Dev.hs b/app/Commands/Dev.hs
@@ -11,7 +11,6 @@ import Commands.Dev.DisplayRoot qualified as DisplayRoot
 import Commands.Dev.Geb qualified as Geb
 import Commands.Dev.Highlight qualified as Highlight
 import Commands.Dev.Internal qualified as Internal
-import Commands.Dev.MiniC qualified as MiniC
 import Commands.Dev.Options
 import Commands.Dev.Parse qualified as Parse
 import Commands.Dev.Runtime qualified as Runtime
@@ -24,7 +23,6 @@ runCommand = \case
   Parse opts -> Parse.runCommand opts
   Scope opts -> Scope.runCommand opts
   Internal opts -> Internal.runCommand opts
-  MiniC opts -> MiniC.runCommand opts
   Termination opts -> Termination.runCommand opts
   Core opts -> Core.runCommand opts
   Geb opts -> Geb.runCommand opts

diff --git a/app/Commands/Dev/Asm/Compile.hs b/app/Commands/Dev/Asm/Compile.hs
@@ -15,26 +15,30 @@ runCommand opts = do
   s <- embed (readFile (toFilePath file))
   case Asm.runParser (toFilePath file) s of
     Left err -> exitJuvixError (JuvixError err)
-    Right tab -> case run $ runError $ asmToMiniC asmOpts tab of
-      Left err -> exitJuvixError err
-      Right C.MiniCResult {..} -> do
-        buildDir <- askBuildDir
-        ensureDir buildDir
-        cFile <- inputCFile file
-        embed $ TIO.writeFile (toFilePath cFile) _resultCCode
-        Compile.runCommand opts {_compileInputFile = AppPath (Abs cFile) False}
+    Right tab -> do
+      tgt <- asmTarget (opts ^. compileTarget)
+      case run $ runError $ asmToMiniC (asmOpts tgt) tab of
+        Left err -> exitJuvixError err
+        Right C.MiniCResult {..} -> do
+          buildDir <- askBuildDir
+          ensureDir buildDir
+          cFile <- inputCFile file
+          embed $ TIO.writeFile (toFilePath cFile) _resultCCode
+          Compile.runCommand opts {_compileInputFile = AppPath (Abs cFile) False}
   where
     getFile :: Sem r (Path Abs File)
     getFile = someBaseToAbs' (opts ^. compileInputFile . pathPath)
 
-    asmOpts :: Asm.Options
-    asmOpts = Asm.makeOptions (asmTarget (opts ^. compileTarget)) (opts ^. compileDebug)
+    asmOpts :: Backend.Target -> Asm.Options
+    asmOpts tgt = Asm.makeOptions tgt (opts ^. compileDebug)
 
-    asmTarget :: CompileTarget -> Backend.Target
+    asmTarget :: CompileTarget -> Sem r Backend.Target
     asmTarget = \case
-      TargetWasm32Wasi -> Backend.TargetCWasm32Wasi
-      TargetNative64 -> Backend.TargetCNative64
-      TargetGeb -> error "GEB target not supported for JuvixAsm"
+      TargetWasm32Wasi -> return Backend.TargetCWasm32Wasi
+      TargetNative64 -> return Backend.TargetCNative64
+      TargetGeb -> exitMsg (ExitFailure 1) "error: GEB target not supported for JuvixAsm"
+      TargetCore -> exitMsg (ExitFailure 1) "error: JuvixCore target not supported for JuvixAsm"
+      TargetAsm -> exitMsg (ExitFailure 1) "error: JuvixAsm target not supported for JuvixAsm"
 
 inputCFile :: (Members '[App] r) => Path Abs File -> Sem r (Path Abs File)
 inputCFile inputFileCompile = do

diff --git a/app/Commands/Dev/Asm/Compile/Options.hs b/app/Commands/Dev/Asm/Compile/Options.hs
@@ -10,4 +10,4 @@ import CommonOptions
 type AsmCompileOptions = CompileOptions
 
 parseAsmCompileOptions :: Parser AsmCompileOptions
-parseAsmCompileOptions = parseCompileOptions
+parseAsmCompileOptions = parseCompileOptions parseInputJuvixAsmFile