Make builds able to depend on external projects (#291)

* Make builds able to depend on external projects

Builds are now able to load external projects and depend on them
as if they were local submodules.

`import $file.external.path.build`

* Disambiguate "dest" for foreign modules.

* Calling modules loaded from external directories "Foreign" to avoid
conflicting with the already existing concept of "ExternalModule".
* Amended the way `dest` is computed for foreign modules
* Added tests to check that the source paths and dest are as expected
* Added a test to show that local modules do not conflict with foreign
modules when they are named the same

* WIP windows build fail

* Added bootstrapping step in CYGWIN CI job

* * Revert externalOutPath deletion
* Add documentation for foreign-modules

* reverting appveyor changes

* Disabling Foreign modules tests against Java9

See #302

.appveyor.yml

@@ -48,4 +48,4 @@ build_script:
       C:\%CYGWIN_DIR%\bin\bash -lc 'chmod +x /usr/local/bin/mill' &&
       C:\%CYGWIN_DIR%\bin\bash -lc "cd /cygdrive/c/mill && mill -i all main.test scalajslib.test")
 
-skip_branch_with_pr: true
+skip_branch_with_pr: true

build.sc

@@ -80,7 +80,7 @@ object core extends MillModule {
   )
 
   def ivyDeps = Agg(
-    ivy"com.lihaoyi:::ammonite:1.1.0-14-037b8eb",
+    ivy"com.lihaoyi:::ammonite:1.1.0-16-147fdfe",
     // Necessary so we can share the JNA classes throughout the build process
     ivy"net.java.dev.jna:jna:4.5.0",
     ivy"net.java.dev.jna:jna-platform:4.5.0"

core/src/mill/define/BaseModule.scala

@@ -6,7 +6,9 @@ object BaseModule{
   case class Implicit(value: BaseModule)
 }
 
-abstract class BaseModule(millSourcePath0: Path, external0: Boolean = false)
+abstract class BaseModule(millSourcePath0: Path,
+                          external0: Boolean = false,
+                          foreign0 : Boolean = false)
                          (implicit millModuleEnclosing0: sourcecode.Enclosing,
                           millModuleLine0: sourcecode.Line,
                           millName0: sourcecode.Name,
@@ -20,6 +22,7 @@ abstract class BaseModule(millSourcePath0: Path, external0: Boolean = false)
       Segments(),
       mill.util.Router.Overrides(0),
       Ctx.External(external0),
+      Ctx.Foreign(foreign0),
       millFile0
     )
   ){
@@ -37,7 +40,7 @@ abstract class BaseModule(millSourcePath0: Path, external0: Boolean = false)
 abstract class ExternalModule(implicit millModuleEnclosing0: sourcecode.Enclosing,
                               millModuleLine0: sourcecode.Line,
                               millName0: sourcecode.Name)
-  extends BaseModule(ammonite.ops.pwd, external0 = true){
+  extends BaseModule(ammonite.ops.pwd, external0 = true, foreign0 = false){
 
   implicit def millDiscoverImplicit: Discover[_] = millDiscover
   assert(

core/src/mill/define/Ctx.scala

@@ -1,7 +1,7 @@
 package mill.define
 
 
-import ammonite.ops.{Path, RelPath}
+import ammonite.ops.Path
 
 import scala.annotation.implicitNotFound
 
@@ -41,6 +41,7 @@ case class Segments(value: Segment*){
       }
       head +: stringSegments
   }
+  def last : Segments = Segments(value.last)
   def render = value.toList match {
     case Nil => ""
     case Segment.Label(head) :: rest =>
@@ -52,6 +53,13 @@ case class Segments(value: Segment*){
   }
 }
 
+object Segments {
+
+  def labels(values : String*) : Segments =
+    Segments(values.map(Segment.Label):_*)
+
+}
+
 @implicitNotFound("Modules, Targets and Commands can only be defined within a mill Module")
 case class Ctx(enclosing: String,
                lineNum: Int,
@@ -60,18 +68,21 @@ case class Ctx(enclosing: String,
                segments: Segments,
                overrides: Int,
                external: Boolean,
+               foreign: Boolean,
                fileName: String){
 }
 
 object Ctx{
   case class External(value: Boolean)
+  case class Foreign(value : Boolean)
   implicit def make(implicit millModuleEnclosing0: sourcecode.Enclosing,
                     millModuleLine0: sourcecode.Line,
                     millName0: sourcecode.Name,
                     millModuleBasePath0: BasePath,
                     segments0: Segments,
                     overrides0: mill.util.Router.Overrides,
                     external0: External,
+                    foreign0: Foreign,
                     fileName: sourcecode.File): Ctx = {
     Ctx(
       millModuleEnclosing0.value,
@@ -81,7 +92,8 @@ object Ctx{
       segments0,
       overrides0.value,
       external0.value,
+      foreign0.value,
       fileName.value
     )
   }
-}
+}

core/src/mill/define/Module.scala

@@ -26,6 +26,7 @@ class Module(implicit outerCtx0: mill.define.Ctx)
   def millOuterCtx = outerCtx0
   def millSourcePath: Path = millOuterCtx.millSourcePath / millOuterCtx.segment.pathSegments
   implicit def millModuleExternal: Ctx.External = Ctx.External(millOuterCtx.external)
+  implicit def millModuleShared: Ctx.Foreign = Ctx.Foreign(millOuterCtx.foreign)
   implicit def millModuleBasePath: BasePath = BasePath(millSourcePath)
   implicit def millModuleSegments: Segments = {
     millOuterCtx.segments ++ Seq(millOuterCtx.segment)

core/src/mill/eval/Evaluator.scala

@@ -124,9 +124,12 @@ case class Evaluator[T](home: Path,
         (newResults, newEvaluated, false)
       case Right(labelledNamedTask) =>
 
+        val out = if (!labelledNamedTask.task.ctx.external) outPath
+          else externalOutPath
+
         val paths = Evaluator.resolveDestPaths(
-          if (!labelledNamedTask.task.ctx.external) outPath else externalOutPath,
-          labelledNamedTask.segments
+          out,
+          destSegments(labelledNamedTask)
         )
 
         if (!exists(paths.out)) mkdir(paths.out)
@@ -191,6 +194,27 @@ case class Evaluator[T](home: Path,
         }
     }
   }
+
+  def destSegments(labelledTask : Labelled[_]) : Segments = {
+    import labelledTask.task.ctx
+    if (ctx.foreign) {
+      val prefix = "foreign-modules"
+      // Computing a path in "out" that uniquely reflects the location
+      // of the foreign module relatively to the current build.
+      val relative = labelledTask.task
+        .ctx.millSourcePath
+        .relativeTo(rootModule.millSourcePath)
+      // Encoding the number of `/..`
+      val ups = if (relative.ups > 0) Segments.labels(s"up-${relative.ups}")
+                else Segments()
+      Segments.labels(prefix)
+        .++(ups)
+        .++(Segments.labels(relative.segments: _*))
+        .++(labelledTask.segments.last)
+    } else labelledTask.segments
+  }
+
+
   def handleTaskResult(v: Any,
                        hashCode: Int,
                        metaPath: Path,

docs/pages/5 - Modules.md

@@ -155,4 +155,41 @@ that is shared by the entire build: for example,
 `mill.scalalib.ScalaWorkerApi/scalaWorker` provides a shared Scala compilation
 service & cache that is shared between all `ScalaModule`s, and
 `mill.scalalib.GenIdea/idea` lets you generate IntelliJ projects without
-needing to define your own `T.command` in your `build.sc` file
+needing to define your own `T.command` in your `build.sc` file
+
+## Foreign Modules 
+
+Mill can load other mill projects from external (or sub) directories, 
+using Ammonite's `$file` magic import, allowing to depend on foreign modules. 
+This allows, for instance, to depend on other projects' sources, or split 
+your build logic into smaller files.
+
+
+For instance, assuming the following stucture : 
+
+```text
+foo/
+    build.sc
+    bar/
+        build.sc 
+baz/
+    build.sc     
+```
+
+you can write the following in `foo/build.sc` : 
+
+```scala
+
+import $file.bar.build
+import $file.^.baz.build
+import mill._ 
+
+def someFoo = T {
+
+    ^.baz.build.someBaz(...)
+    bar.build.someBar(...) 
+    ...
+}
+```
+
+The output of the foreign tasks will be cached under `foo/out/foreign-modules/`. 

main/src/mill/main/MainRunner.scala

@@ -1,9 +1,10 @@
 package mill.main
-import java.io.{InputStream, OutputStream, PrintStream}
+import java.io.{InputStream, PrintStream}
 
 import ammonite.Main
 import ammonite.interp.{Interpreter, Preprocessor}
 import ammonite.ops.Path
+import ammonite.util.Util.CodeSource
 import ammonite.util._
 import mill.eval.{Evaluator, PathRef}
 import mill.util.PrintLogger
@@ -120,20 +121,26 @@ class MainRunner(val config: ammonite.main.Cli.Config,
 
   object CustomCodeWrapper extends Preprocessor.CodeWrapper {
     def apply(code: String,
-              pkgName: Seq[ammonite.util.Name],
+              source: CodeSource,
               imports: ammonite.util.Imports,
               printCode: String,
               indexedWrapperName: ammonite.util.Name,
               extraCode: String): (String, String, Int) = {
+      import source.pkgName
       val wrapName = indexedWrapperName.backticked
-      val literalPath = pprint.Util.literalize(config.wd.toString)
+      val path = source
+        .path
+        .map(path => path.toNIO.getParent)
+        .getOrElse(config.wd.toNIO)
+      val literalPath = pprint.Util.literalize(path.toString)
+      val external = !(path.compareTo(config.wd.toNIO) == 0)
       val top = s"""
         |package ${pkgName.head.encoded}
         |package ${Util.encodeScalaSourcePath(pkgName.tail)}
         |$imports
         |import mill._
         |object $wrapName
-        |extends mill.define.BaseModule(ammonite.ops.Path($literalPath))
+        |extends mill.define.BaseModule(ammonite.ops.Path($literalPath), foreign0 = $external)
         |with $wrapName{
         |  // Stub to make sure Ammonite has something to call after it evaluates a script,
         |  // even if it does nothing...

main/test/resources/examples/foreign/conflict/build.sc

@@ -0,0 +1,24 @@
+import $file.inner.{build => innerBuild}
+import mill._
+import ammonite.ops._
+
+// In this build, we have a local module targeting
+// the 'inner sub-directory, and an imported foreign
+// module in that same directory. Their sourcePaths
+// should be the same, but their dest paths should
+// be different to avoid both modules over-writing
+// each other's caches .
+
+def checkPaths : T[Unit] = T {
+  if (innerBuild.millSourcePath != inner.millSourcePath)
+    throw new Exception("Source paths should be the same")
+}
+
+def checkDests : T[Unit] = T {
+  if (innerBuild.selfDest == inner.selfDest)
+    throw new Exception("Dest paths should be different")
+}
+
+object inner extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}

main/test/resources/examples/foreign/conflict/inner/build.sc

@@ -0,0 +1,4 @@
+import mill._
+import ammonite.ops._
+
+def selfDest = T { T.ctx().dest / up / up }

main/test/resources/examples/foreign/outer/build.sc

@@ -0,0 +1,16 @@
+import $file.inner.build
+import mill._
+import ammonite.ops._
+
+trait PathAware extends mill.Module {
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
+object sub extends PathAware with DestAware {
+  object sub extends PathAware with DestAware
+}
+

main/test/resources/examples/foreign/outer/inner/build.sc

@@ -0,0 +1,15 @@
+import mill._
+import ammonite.ops._
+
+trait PathAware extends mill.Module {
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+
+object sub extends PathAware with DestAware {
+  object sub extends PathAware with DestAware
+}
+

main/test/resources/examples/foreign/project/build.sc

@@ -0,0 +1,82 @@
+import $file.^.outer.build
+import $file.inner.build
+
+import ammonite.ops._
+import mill._
+
+def assertPaths(p1 : Path, p2 : Path) : Unit = if (p1 != p2) throw new Exception(
+  s"Paths were not equal : \n- $p1 \n- $p2"
+)
+
+object sub extends PathAware with DestAware {
+
+  object sub extends PathAware with DestAware
+
+  object sub2 extends ^.outer.build.PathAware with ^.outer.build.DestAware
+
+}
+
+def checkProjectPaths = T {
+  val thisPath : Path = millSourcePath
+  assert(thisPath.last == "project")
+  assertPaths(sub.selfPath(), thisPath / 'sub)
+  assertPaths(sub.sub.selfPath(), thisPath / 'sub / 'sub)
+  assertPaths(sub.sub2.selfPath(), thisPath / 'sub / 'sub2)
+}
+
+def checkInnerPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(inner.build.millSourcePath, thisPath / 'inner )
+  assertPaths(inner.build.sub.selfPath(), thisPath / 'inner / 'sub)
+  assertPaths(inner.build.sub.sub.selfPath(), thisPath / 'inner / 'sub / 'sub)
+}
+
+def checkOuterPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(^.outer.build.millSourcePath, thisPath / up / 'outer )
+  assertPaths(^.outer.build.sub.selfPath(), thisPath / up / 'outer / 'sub)
+  assertPaths(^.outer.build.sub.sub.selfPath(), thisPath / up / 'outer / 'sub / 'sub)
+}
+
+def checkOuterInnerPaths = T {
+  val thisPath : Path = millSourcePath
+  assertPaths(^.outer.inner.build.millSourcePath, thisPath / up / 'outer / 'inner )
+  assertPaths(^.outer.inner.build.sub.selfPath(), thisPath / up / 'outer / 'inner /'sub)
+  assertPaths(^.outer.inner.build.sub.sub.selfPath(), thisPath / up / 'outer / 'inner / 'sub / 'sub)
+}
+
+def checkProjectDests = T {
+  val outPath : Path = millSourcePath / 'out
+  assertPaths(sub.selfDest(), outPath / 'sub)
+  assertPaths(sub.sub.selfDest(), outPath / 'sub / 'sub)
+  assertPaths(sub.sub2.selfDest(), outPath / 'sub / 'sub2)
+}
+
+def checkInnerDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(inner.build.sub.selfDest(), foreignOut / 'inner / 'sub)
+  assertPaths(inner.build.sub.sub.selfDest(), foreignOut / 'inner / 'sub / 'sub)
+}
+
+def checkOuterDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(^.outer.build.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'sub )
+  assertPaths(^.outer.build.sub.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'sub / 'sub)
+}
+
+def checkOuterInnerDests = T {
+  val foreignOut : Path = millSourcePath / 'out / "foreign-modules"
+  assertPaths(^.outer.inner.build.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'inner / 'sub)
+  assertPaths(^.outer.inner.build.sub.sub.selfDest(), foreignOut / "up-1" / 'outer/ 'inner / 'sub / 'sub)
+}
+
+
+trait PathAware extends mill.Module {
+
+  def selfPath = T { millSourcePath }
+}
+
+trait DestAware extends mill.Module {
+  def selfDest = T { T.ctx().dest / up / up }
+}
+