Bytecode reachability analysis for fine-grained target invalidation

bsp/worker/src/mill/bsp/worker/MillBuildServer.scala

@@ -102,7 +102,14 @@ private class MillBuildServer(
     if (statePromise.isCompleted) statePromise = Promise[State]() // replace the promise
     evaluatorOpt = evaluator
     evaluatorOpt.foreach(e =>
-      statePromise.success(new State(e.rootModule.millSourcePath, e.baseLogger, debug))
+      statePromise.success(
+        new State(
+          e.rootModule.millSourcePath,
+          e.baseLogger,
+          debug,
+          e.disableCallgraphInvalidation
+        )
+      )
     )
   }
 

bsp/worker/src/mill/bsp/worker/State.scala

@@ -7,7 +7,12 @@ import mill.scalalib.internal.{JavaModuleUtils, ModuleUtils}
 import mill.define.Module
 import mill.util.ColorLogger
 
-private class State(projectRoot: os.Path, baseLogger: ColorLogger, debug: String => Unit) {
+private class State(
+    projectRoot: os.Path,
+    baseLogger: ColorLogger,
+    debug: String => Unit,
+    disableCallgraphInvalidation: Boolean
+) {
   lazy val bspModulesById: Map[BuildTargetIdentifier, BspModule] = {
     val modules: Seq[(Module, Seq[Module])] = rootModules
       .map(rootModule => (rootModule, JavaModuleUtils.transitiveModules(rootModule)))
@@ -39,7 +44,8 @@ private class State(projectRoot: os.Path, baseLogger: ColorLogger, debug: String
       threadCount = None,
       targetsAndParams = Seq("resolve", "_"),
       prevRunnerState = mill.runner.RunnerState.empty,
-      logger = baseLogger
+      logger = baseLogger,
+      disableCallgraphInvalidation = disableCallgraphInvalidation
     ).evaluate()
 
     val rootModules0 = evaluated.result.frames

build.sc

@@ -1,4 +1,5 @@
 // plugins and dependencies
+
 import $file.ci.shared
 import $file.ci.upload
 import $ivy.`org.scalaj::scalaj-http:2.4.2`
@@ -356,6 +357,13 @@ trait MillPublishScalaModule extends MillScalaModule with MillPublishJavaModule
 
 /** Publishable module which contains strictly handled API. */
 trait MillStableScalaModule extends MillPublishScalaModule with Mima {
+  import com.github.lolgab.mill.mima._
+  override def mimaBinaryIssueFilters: T[Seq[ProblemFilter]] = Seq(
+    ProblemFilter.exclude[Problem]("mill.eval.ProfileLogger*"),
+    ProblemFilter.exclude[Problem]("mill.eval.GroupEvaluator*"),
+    ProblemFilter.exclude[Problem]("mill.eval.Tarjans*"),
+    ProblemFilter.exclude[Problem]("mill.define.Ctx#Impl*"),
+  )
   def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions
 
   def mimaPreviousArtifacts: T[Agg[Dep]] = T {
@@ -455,6 +463,7 @@ object main extends MillStableScalaModule with BuildInfo {
       BuildInfo.Value("millVersion", millVersion(), "Mill version."),
       BuildInfo.Value("millDocUrl", Settings.docUrl, "Mill documentation url.")
     )
+
     def ivyDeps = Agg(
       Deps.osLib,
       Deps.upickle,
@@ -463,10 +472,65 @@ object main extends MillStableScalaModule with BuildInfo {
       Deps.sbtTestInterface
     )
   }
+
   object util extends MillStableScalaModule {
     def moduleDeps = Seq(api, client)
     def ivyDeps = Agg(Deps.coursier, Deps.jline)
   }
+
+  object codesig extends MillPublishScalaModule {
+    override def ivyDeps = Agg(ivy"org.ow2.asm:asm-tree:9.5", Deps.osLib, ivy"com.lihaoyi::pprint:0.8.1")
+    def moduleDeps = Seq(util)
+
+    override lazy val test: CodeSigTests = new CodeSigTests{}
+    trait CodeSigTests extends MillScalaTests{
+      val caseKeys = interp.watchValue(
+        os.walk(millSourcePath / "cases", maxDepth = 3)
+          .map(_.subRelativeTo(millSourcePath / "cases").segments)
+          .collect{case Seq(a, b, c) => s"$a-$b-$c"}
+      )
+
+      def testLogFolder = T{ T.dest }
+
+      def caseEnvs[V](f1: CaseModule => Task[V])(s: String, f2: V => String) = {
+        T.traverse(caseKeys) { i => f1(cases(i)).map(v => s"MILL_TEST_${s}_$i" -> f2(v)) }
+      }
+      def forkEnv = T{
+        Map("MILL_TEST_LOGS" -> testLogFolder().toString) ++
+        caseEnvs(_.compile)("CLASSES", _.classes.path.toString)() ++
+        caseEnvs(_.compileClasspath)("CLASSPATH", _.map(_.path).mkString(","))() ++
+        caseEnvs(_.sources)("SOURCES", _.head.path.toString)()
+      }
+
+      object cases extends Cross[CaseModule](caseKeys)
+      trait CaseModule extends ScalaModule with Cross.Module[String]{
+        def caseName = crossValue
+        object external extends ScalaModule{
+          def scalaVersion = "2.13.10"
+        }
+
+        def moduleDeps = Seq(external)
+
+        val Array(prefix, suffix, rest) = caseName.split("-", 3)
+        def millSourcePath = super.millSourcePath / prefix / suffix / rest
+        def scalaVersion = "2.13.10"
+        def ivyDeps = T{
+          if (!caseName.contains("realistic") && !caseName.contains("sourcecode")) super.ivyDeps()
+          else Agg(
+            ivy"com.lihaoyi::fastparse:3.0.1",
+            ivy"com.lihaoyi::scalatags:0.12.0",
+            ivy"com.lihaoyi::cask:0.9.1",
+            ivy"com.lihaoyi::castor:0.1.7",
+            Deps.mainargs,
+            Deps.requests,
+            Deps.osLib,
+            Deps.upickle,
+          )
+        }
+      }
+    }
+  }
+
   object define extends MillStableScalaModule {
     def moduleDeps = Seq(api, util)
     def compileIvyDeps = Agg(Deps.scalaReflect(scalaVersion()))
@@ -1172,7 +1236,7 @@ def launcherScript(
 }
 
 object runner extends MillPublishScalaModule {
-  def moduleDeps = Seq(scalalib, scalajslib, scalanativelib, bsp, linenumbers)
+  def moduleDeps = Seq(scalalib, scalajslib, scalanativelib, bsp, linenumbers, main.codesig)
   def skipPreviousVersions: T[Seq[String]] = Seq("0.11.0-M7")
 
   object linenumbers extends MillPublishScalaModule {

integration/feature/codesig-hello/repo/build.sc

@@ -0,0 +1,6 @@
+import mill._
+
+val valueFoo = 0
+val valueFooUsedInBar = 0
+def helperFoo = { println("running helperFoo"); 1 + valueFoo }
+def foo = T{ println("running foo"); helperFoo }

integration/feature/codesig-hello/test/src/CodeSigHelloTests.scala

@@ -0,0 +1,55 @@
+package mill.integration
+
+import utest._
+
+object CodeSigHelloTests extends IntegrationTestSuite {
+  val tests = Tests {
+    val wsRoot = initWorkspace()
+    "simple" - {
+      // Make sure the simplest case where we have a single target calling a single helper
+      // method is properly invalidated when either the target body, or the helper method's body
+      // is changed, or something changed in the constructor
+      val initial = evalStdout("foo")
+
+      assert(initial.out.linesIterator.toSeq == Seq("running foo", "running helperFoo"))
+
+      val cached = evalStdout("foo")
+      assert(cached.out == "")
+
+      mangleFile(wsRoot / "build.sc", _.replace("running foo", "running foo2"))
+      val mangledFoo = evalStdout("foo")
+
+      assert(mangledFoo.out.linesIterator.toSeq == Seq("running foo2", "running helperFoo"))
+
+      val cached2 = evalStdout("foo")
+      assert(cached2.out == "")
+
+      mangleFile(wsRoot / "build.sc", _.replace("running helperFoo", "running helperFoo2"))
+      val mangledHelperFoo = evalStdout("foo")
+
+      assert(mangledHelperFoo.out.linesIterator.toSeq == Seq("running foo2", "running helperFoo2"))
+
+      // Make sure changing `val`s, which only affects the Module constructor and
+      // not the Target method itself, causes invalidation
+      mangleFile(wsRoot / "build.sc", _.replace("val valueFoo = 0", "val valueFoo = 10"))
+      val mangledValFoo = evalStdout("foo")
+      assert(mangledValFoo.out.linesIterator.toSeq == Seq("running foo2", "running helperFoo2"))
+
+      // Even modifying `val`s that do not affect the target invalidates it, because
+      // we only know that the constructor changed and don't do enough analysis to
+      // know that this particular val is not used
+      mangleFile(
+        wsRoot / "build.sc",
+        _.replace("val valueFooUsedInBar = 0", "val valueFooUsedInBar = 10")
+      )
+      val mangledValFooUsedInBar = evalStdout("foo")
+      assert(mangledValFooUsedInBar.out.linesIterator.toSeq == Seq(
+        "running foo2",
+        "running helperFoo2"
+      ))
+
+      val cached3 = evalStdout("foo")
+      assert(cached3.out == "")
+    }
+  }
+}

integration/feature/codesig-nested/repo/build.sc

@@ -0,0 +1,39 @@
+import mill._
+
+val valueFoo = 0
+val valueFooUsedInBar = 0
+def helperFoo = { println("running helperFoo"); 1 + valueFoo }
+def foo = T{ println("running foo"); helperFoo }
+
+object outer extends Module{
+  val valueBar = 0
+  val valueBarUsedInQux = 0
+  def helperBar = { println("running helperBar"); 20 }
+  def bar = T{ println("running bar"); helperBar + valueBar + valueFooUsedInBar }
+
+  trait InnerModule extends Module{
+    val valueQux = 0
+    def helperQux = { println("running helperQux"); 300 + valueQux + valueBarUsedInQux }
+    def qux = T{ println("running qux"); foo() + bar() + helperQux }
+  }
+  object inner extends InnerModule
+}
+
+trait TraitOuter extends Module{
+  val valueTraitOuter = 0
+  def helperTraitOuter = { println("running helperTraitOuter"); 300 + valueTraitOuter }
+  def outer = T{ println("running outer"); foo() + helperTraitOuter }
+
+  object traitInner extends Module{
+    val valueTraitInner = 0
+    def helperTraitInner = { println("running helperTraitInner"); 300 + valueTraitOuter + valueTraitInner }
+    def inner = T{ println("running inner"); foo() + outer() + helperTraitInner }
+  }
+}
+
+object traitOuter extends TraitOuter
+
+object crossOuter extends Cross[CrossOuter](1337, 31337, 31415926)
+trait CrossOuter extends TraitOuter with Cross.Module[Int]{
+  override val valueTraitOuter = crossValue
+}