Make mill-build folder opt in via import $mill.build (#2527)

Fixes #2461

For now it remains hard-coded as `mill-build/`, for simplicity. Given
the `mill-` prefix, it's unlikely to collide with user-defined folders.
Making it configurable would open another can of worms where the `import
$meta.foo` can now collide with an `object foo`, which is very
unintuitive.

For now, we just parse the script files one additional time in
`MillBuildBootstrap.scala`. This is a bit wasteful, but is probably fast
enough for now, and we're already pretty sloppy parsing everything twice
in `MillBuildRootModule#scriptSources` and
`MillBuildRootModule#parseBuildFiles`, so parsing things three times
isn't the end of the world. We can look into optimizing it in future if
necessary

example/misc/4-mill-build-folder/build.sc

@@ -1,3 +1,4 @@
+import $meta._
 import mill._, scalalib._
 import scalatags.Text.all._
 
@@ -20,7 +21,7 @@ object foo extends RootModule with ScalaModule {
 // file and it's `import $file` and `$ivy` are a shorthand syntax for defining
 // a Mill `ScalaModule`, with sources and `ivyDeps` and so on, which is
 // compiled and executed to perform your build. This module lives in
-// `mill-build/`.
+// `mill-build/`, and can be enabled via the `import $meta._` statement above.
 
 /** See Also: mill-build/build.sc */
 

integration/failure/invalid-meta-module/repo/build.sc

@@ -0,0 +1 @@
+import $meta._

integration/feature/editing/repo/build.sc

@@ -1,3 +1,4 @@
+import $meta._
 import mill._, scalalib._
 import scalatags.Text.all._
 

integration/feature/editing/repo/mill-build/build.sc

@@ -1,3 +1,4 @@
+import $meta._
 import mill._, scalalib._
 
 object millbuild extends MillBuildRootModule{

integration/feature/editing/test/src/MultiLevelBuildTests.scala

@@ -48,7 +48,8 @@ object MultiLevelBuildTests extends IntegrationTestSuite {
       for (depth <- Range(0, n))
         yield {
           val path = wsRoot / "out" / Seq.fill(depth)("mill-build") / "mill-runner-state.json"
-          upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path
+          if (os.exists(path)) upickle.default.read[RunnerState.Frame.Logged](os.read(path)) -> path
+          else RunnerState.Frame.Logged(Map(), Seq(), Seq(), Map(), None, Seq(), 0) -> path
         }
     }
 
@@ -69,14 +70,18 @@ object MultiLevelBuildTests extends IntegrationTestSuite {
       }
     }
 
-    def evalCheckErr(expected: String*) = {
+    def evalCheckErr(expectedSnippets: String*) = {
+      // Wipe out stale state files to make sure they don't get picked up when
+      // Mill aborts early and fails to generate a new one
+      os.walk(wsRoot / "out").filter(_.last == "mill-runner-state.json").foreach(os.remove(_))
+
       val res = evalStdout("foo.run")
       assert(res.isSuccess == false)
       // Prepend a "\n" to allow callsites to use "\n" to test for start of
       // line, even though the first line doesn't have a "\n" at the start
       val err = "\n" + res.err
-      for (e <- expected) {
-        assert(err.contains(e))
+      for (expected <- expectedSnippets) {
+        assert(err.contains(expected))
       }
     }
 
@@ -189,17 +194,23 @@ object MultiLevelBuildTests extends IntegrationTestSuite {
         "\n1 targets failed",
         "\ngenerateScriptSources build.sc"
       )
-      checkWatchedFiles(Nil, buildPaths, buildPaths2, buildPaths3)
-      checkChangedClassloaders(null, null, false, false)
+      checkWatchedFiles(Nil, buildPaths, Nil, Nil)
+      // When one of the meta-builds still has parse errors, all classloaders
+      // remain null, because none of the meta-builds can evaluate. Only once
+      // all of them parse successfully do we get a new set of classloaders for
+      // every level of the meta-build
+      checkChangedClassloaders(null, null, null, null)
 
+      fixParseError(wsRoot / "build.sc")
       causeParseError(wsRoot / "mill-build" / "build.sc")
       evalCheckErr(
         "\n1 targets failed",
         "\ngenerateScriptSources mill-build/build.sc"
       )
-      checkWatchedFiles(Nil, Nil, buildPaths2, buildPaths3)
-      checkChangedClassloaders(null, null, null, false)
+      checkWatchedFiles(Nil, Nil, buildPaths2, Nil)
+      checkChangedClassloaders(null, null, null, null)
 
+      fixParseError(wsRoot / "mill-build" / "build.sc")
       causeParseError(wsRoot / "mill-build" / "mill-build" / "build.sc")
       evalCheckErr(
         "\n1 targets failed",
@@ -209,25 +220,27 @@ object MultiLevelBuildTests extends IntegrationTestSuite {
       checkChangedClassloaders(null, null, null, null)
 
       fixParseError(wsRoot / "mill-build" / "mill-build" / "build.sc")
+      causeParseError(wsRoot / "mill-build" / "build.sc")
       evalCheckErr(
         "\n1 targets failed",
         "\ngenerateScriptSources mill-build/build.sc"
       )
-      checkWatchedFiles(Nil, Nil, buildPaths2, buildPaths3)
-      checkChangedClassloaders(null, null, null, true)
+      checkWatchedFiles(Nil, Nil, buildPaths2, Nil)
+      checkChangedClassloaders(null, null, null, null)
 
       fixParseError(wsRoot / "mill-build" / "build.sc")
+      causeParseError(wsRoot / "build.sc")
       evalCheckErr(
         "\n1 targets failed",
         "\ngenerateScriptSources build.sc"
       )
-      checkWatchedFiles(Nil, buildPaths, buildPaths2, buildPaths3)
-      checkChangedClassloaders(null, null, true, false)
+      checkWatchedFiles(Nil, buildPaths, Nil, Nil)
+      checkChangedClassloaders(null, null, null, null)
 
       fixParseError(wsRoot / "build.sc")
       runAssertSuccess("<h1>hello</h1><p>world</p><p>0.8.2</p>!")
       checkWatchedFiles(fooPaths, buildPaths, buildPaths2, buildPaths3)
-      checkChangedClassloaders(null, true, false, false)
+      checkChangedClassloaders(null, true, true, true)
     }
 
     test("compileErrorEdits") {

runner/src/mill/runner/FileImportGraph.scala

@@ -10,7 +10,8 @@ case class FileImportGraph(
     repos: Seq[(String, os.Path)],
     ivyDeps: Set[String],
     importGraphEdges: Map[os.Path, Seq[os.Path]],
-    errors: Seq[String]
+    errors: Seq[String],
+    millImport: Boolean
 )
 
 /**
@@ -33,6 +34,7 @@ object FileImportGraph {
     val seenRepo = mutable.ListBuffer.empty[(String, os.Path)]
     val importGraphEdges = mutable.Map.empty[os.Path, Seq[os.Path]]
     val errors = mutable.Buffer.empty[String]
+    var millImport = false
 
     def walkScripts(s: os.Path): Unit = {
       importGraphEdges(s) = Nil
@@ -84,6 +86,9 @@ object FileImportGraph {
           case ImportTree(Seq(("$ivy", _), rest @ _*), mapping, start, end) =>
             seenIvy.addAll(mapping.map(_._1))
             (start, "_root_._", end)
+          case ImportTree(Seq(("$meta", _), rest @ _*), mapping, start, end) =>
+            millImport = true
+            (start, "_root_._", end)
           case ImportTree(Seq(("$file", _), rest @ _*), mapping, start, end) =>
             val nextPaths = mapping.map { case (lhs, rhs) => nextPathFor(s, rest.map(_._1) :+ lhs) }
 
@@ -113,7 +118,8 @@ object FileImportGraph {
       seenRepo.toSeq,
       seenIvy.toSet,
       importGraphEdges.toMap,
-      errors.toSeq
+      errors.toSeq,
+      millImport
     )
   }
 

runner/src/mill/runner/MillBuildBootstrap.scala

@@ -64,11 +64,20 @@ class MillBuildBootstrap(
     val prevOuterFrameOpt = prevRunnerState.frames.lift(depth - 1)
 
     val nestedRunnerState =
-      if (!os.exists(recRoot(projectRoot, depth) / "build.sc")) {
-        if (depth == 0) {
-          RunnerState(None, Nil, Some("build.sc file not found. Are you in a Mill project folder?"))
-        } else {
+      if (depth == 0) {
+        if (os.exists(recRoot(projectRoot, depth) / "build.sc")) evaluateRec(depth + 1)
+        else {
+          val msg = "build.sc file not found. Are you in a Mill project folder?"
+          RunnerState(None, Nil, Some(msg))
+        }
+      } else {
+        val parsedScriptFiles = FileImportGraph.parseBuildFiles(
+          projectRoot,
+          recRoot(projectRoot, depth) / os.up
+        )
 
+        if (parsedScriptFiles.millImport) evaluateRec(depth + 1)
+        else {
           val bootstrapModule =
             new MillBuildRootModule.BootstrapModule(
               projectRoot,
@@ -83,8 +92,6 @@ class MillBuildBootstrap(
             )
           RunnerState(Some(bootstrapModule), Nil, None)
         }
-      } else {
-        evaluateRec(depth + 1)
       }
 
     val res = if (nestedRunnerState.errorOpt.isDefined) {