Support customizing coursier dependency resolution (make OS specific …

…classifiers work) (#775)

Added CoursierModule.resolutionCustomizer task.

This helps in customizing coursier resolution process, e.g.
to add OS or JDK specific resolution properties sometimes used by Maven

Pull request: #775

main/src/modules/Jvm.scala

@@ -474,6 +474,7 @@ object Jvm {
     PathRef(outputPath)
   }
 
+
   /**
     * Resolve dependencies using Coursier.
     *
@@ -486,10 +487,11 @@ object Jvm {
                           force: IterableOnce[coursier.Dependency],
                           sources: Boolean = false,
                           mapDependencies: Option[Dependency => Dependency] = None,
-                          ctx: Option[mill.util.Ctx.Log] = None): Result[Agg[PathRef]] = {
+                          customizer: Option[coursier.core.Resolution => coursier.core.Resolution] = None,
+                          ctx: Option[mill.api.Ctx.Log] = None): Result[Agg[PathRef]] = {
 
     val (_, resolution) = resolveDependenciesMetadata(
-      repositories, deps, force, mapDependencies, ctx
+      repositories, deps, force, mapDependencies, customizer, ctx
     )
     val errs = resolution.errors
 
@@ -552,12 +554,12 @@ object Jvm {
     }
   }
 
-
   def resolveDependenciesMetadata(repositories: Seq[Repository],
                                   deps: IterableOnce[coursier.Dependency],
                                   force: IterableOnce[coursier.Dependency],
                                   mapDependencies: Option[Dependency => Dependency] = None,
-                                  ctx: Option[mill.util.Ctx.Log] = None) = {
+                                  customizer: Option[coursier.core.Resolution => coursier.core.Resolution] = None,
+                                  ctx: Option[mill.api.Ctx.Log] = None): (Seq[Dependency], Resolution) = {
 
     val cachePolicies = coursier.cache.CacheDefaults.cachePolicies
 
@@ -566,11 +568,13 @@ object Jvm {
       .map{d => d.module -> d.version}
       .toMap
 
-    val start = Resolution()
-      .withRootDependencies(deps.map(mapDependencies.getOrElse(identity[Dependency](_))).toSeq)
+    val start0 = Resolution()
+      .withRootDependencies(deps.iterator.map(mapDependencies.getOrElse(identity[Dependency](_))).toSeq)
       .withForceVersions(forceVersions)
       .withMapDependencies(mapDependencies)
 
+    val start = customizer.getOrElse(identity[Resolution](_)).apply(start0)
+
     val resolutionLogger = ctx.map(c => new TickerResolutionLogger(c))
     val cache = resolutionLogger match {
       case None => coursier.cache.FileCache[Task].withCachePolicies(cachePolicies)
@@ -673,4 +677,37 @@ object Jvm {
       manifest
     }
   }
+
+  @deprecated("Use alternative overload. This one is only for binary backwards compatibility.", "mill after 0.9.6")
+  def resolveDependencies(repositories: Seq[Repository],
+                          deps: IterableOnce[coursier.Dependency],
+                          force: IterableOnce[coursier.Dependency],
+                          sources: Boolean,
+                          mapDependencies: Option[Dependency => Dependency],
+                          ctx: Option[mill.api.Ctx.Log]): Result[Agg[PathRef]] =
+    resolveDependencies(
+      repositories = repositories,
+      deps = deps,
+      force = force,
+      sources = sources,
+      mapDependencies = mapDependencies,
+      customizer = None,
+      ctx = ctx
+    )
+
+  @deprecated("Use alternative overload. This one is only for binary backwards compatibility.", "mill after 0.9.6")
+  def resolveDependenciesMetadata(repositories: Seq[Repository],
+                                  deps: IterableOnce[coursier.Dependency],
+                                  force: IterableOnce[coursier.Dependency],
+                                  mapDependencies: Option[Dependency => Dependency],
+                                  ctx: Option[mill.api.Ctx.Log]): (Seq[Dependency], Resolution) =
+    resolveDependenciesMetadata(
+      repositories = repositories,
+      deps = deps,
+      force = force,
+      mapDependencies = mapDependencies,
+      customizer = None,
+      ctx = ctx
+    )
+
 }

scalalib/src/CoursierModule.scala

@@ -1,6 +1,7 @@
 package mill.scalalib
 
 import coursier.{Dependency, Repository, Resolve}
+import coursier.core.{Resolution}
 import mill.{Agg, T}
 import mill.define.Task
 import mill.api.PathRef
@@ -26,30 +27,50 @@ trait CoursierModule extends mill.Module {
     */
   def resolveDeps(deps: Task[Agg[Dep]], sources: Boolean = false): Task[Agg[PathRef]] = T.task {
     Lib.resolveDependencies(
-      repositoriesTask(),
-      resolveCoursierDependency().apply(_),
-      deps(),
-      sources,
+      repositories = repositoriesTask(),
+      depToDependency = resolveCoursierDependency().apply(_),
+      deps = deps(),
+      sources = sources,
       mapDependencies = Some(mapDependencies()),
-      Some(implicitly[mill.util.Ctx.Log])
+      customizer = resolutionCustomizer(),
+      ctx = Some(implicitly[mill.api.Ctx.Log])
     )
   }
 
   /**
     * Map dependencies before resolving them.
     * Override this to customize the set of dependencies.
     */
-  def mapDependencies: Task[Dependency => Dependency] = T.task { d: coursier.Dependency => d }
+  def mapDependencies: Task[Dependency => Dependency] = T.task { d: Dependency => d }
 
   /**
     * The repositories used to resolved dependencies with [[resolveDeps()]].
     */
-  @deprecated("Use repositoriesTask instead", "after mill 0.8.0")
-  def repositories: Seq[Repository] = Resolve.defaultRepositories
+  def repositoriesTask: Task[Seq[Repository]] = T.task { repositories }
+
+  /**
+    * Customize the coursier resolution resolution process.
+    * This is rarely needed to changed, as the default try to provide a
+    * highly reproducible resolution process. But sometime, you need
+    * more control, e.g. you want to add some OS or JDK specific resolution properties
+    * which are sometimes used by Maven and therefore found in dependency artifact metadata.
+    * For example, the JavaFX artifacts are known to use OS specific properties.
+    * To fix resolution for JavaFX, you could override this task like the following:
+    * {{{
+    *     override def resolutionCustomizer = T.task {
+    *       Some( (r: coursier.core.Resolution) =>
+    *         r.withOsInfo(coursier.core.Activation.Os.fromProperties(sys.props.toMap))
+    *       )
+    *     }
+    * }}}
+    * @return
+    */
+  def resolutionCustomizer: Task[Option[Resolution => Resolution]] = T.task { None }
 
   /**
     * The repositories used to resolved dependencies with [[resolveDeps()]].
     */
-  def repositoriesTask: Task[Seq[Repository]] = T.task { repositories }
+  @deprecated("Use repositoriesTask instead", "after mill 0.8.0")
+  def repositories: Seq[Repository] = Resolve.defaultRepositories
 
 }

scalalib/src/JavaModule.scala

@@ -444,7 +444,8 @@ trait JavaModule
         repositoriesTask(),
         resolveCoursierDependency().apply(_),
         additionalDeps() ++ transitiveIvyDeps(),
-        Some(mapDependencies())
+        Some(mapDependencies()),
+        customizer = resolutionCustomizer()
       )
 
       println(

scalalib/src/Lib.scala

@@ -11,7 +11,7 @@ import mill.api.{Ctx, Loose, PathRef, Result}
 import sbt.testing._
 
 
-object Lib{
+object Lib {
   def depToDependencyJava(dep: Dep, platformSuffix: String = ""): Dependency = {
     assert(dep.cross.isConstant, s"Not a Java dependency: $dep")
     depToDependency(dep, "", platformSuffix)
@@ -28,16 +28,19 @@ object Lib{
                                   depToDependency: Dep => coursier.Dependency,
                                   deps: IterableOnce[Dep],
                                   mapDependencies: Option[Dependency => Dependency] = None,
+                                  customizer: Option[coursier.core.Resolution => coursier.core.Resolution] = None,
                                   ctx: Option[Ctx.Log] = None): (Seq[Dependency], Resolution) = {
-    val depSeq = deps.toSeq
+    val depSeq = deps.iterator.toSeq
     mill.modules.Jvm.resolveDependenciesMetadata(
-      repositories,
-      depSeq.map(depToDependency),
-      depSeq.filter(_.force).map(depToDependency),
-      mapDependencies,
-      ctx
+      repositories = repositories,
+      deps = depSeq.map(depToDependency),
+      force = depSeq.filter(_.force).map(depToDependency),
+      mapDependencies = mapDependencies,
+      customizer = customizer,
+      ctx = ctx
     )
   }
+
   /**
     * Resolve dependencies using Coursier.
     *
@@ -50,17 +53,20 @@ object Lib{
                           deps: IterableOnce[Dep],
                           sources: Boolean = false,
                           mapDependencies: Option[Dependency => Dependency] = None,
+                          customizer: Option[coursier.core.Resolution => coursier.core.Resolution] = None,
                           ctx: Option[Ctx.Log] = None): Result[Agg[PathRef]] = {
-    val depSeq = deps.toSeq
+    val depSeq = deps.iterator.toSeq
     mill.modules.Jvm.resolveDependencies(
-      repositories,
-      depSeq.map(depToDependency),
-      depSeq.filter(_.force).map(depToDependency),
-      sources,
-      mapDependencies,
-      ctx
+      repositories = repositories,
+      deps = depSeq.map(depToDependency),
+      force = depSeq.filter(_.force).map(depToDependency),
+      sources = sources,
+      mapDependencies = mapDependencies,
+      customizer = customizer,
+      ctx = ctx
     )
   }
+
   def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String): Loose.Agg[Dep] =
     if (mill.scalalib.api.Util.isDotty(scalaVersion))
       Agg(
@@ -165,4 +171,36 @@ object Lib{
     }.map { f => (cls, f) }
   }
 
+  @deprecated("User other overload instead. Only for binary backward compatibility.", "mill after 0.9.6")
+  def resolveDependenciesMetadata(repositories: Seq[Repository],
+                                  depToDependency: Dep => coursier.Dependency,
+                                  deps: IterableOnce[Dep],
+                                  mapDependencies: Option[Dependency => Dependency],
+                                  ctx: Option[Ctx.Log]): (Seq[Dependency], Resolution) =
+    resolveDependenciesMetadata(
+      repositories = repositories,
+      depToDependency = depToDependency,
+      deps = deps,
+      mapDependencies = mapDependencies,
+      customizer = None,
+      ctx = ctx
+    )
+
+  @deprecated("User other overload instead. Only for binary backward compatibility.", "mill after 0.9.6")
+  def resolveDependencies(repositories: Seq[Repository],
+                          depToDependency: Dep => coursier.Dependency,
+                          deps: IterableOnce[Dep],
+                          sources: Boolean,
+                          mapDependencies: Option[Dependency => Dependency],
+                          ctx: Option[Ctx.Log]): Result[Agg[PathRef]] =
+    resolveDependencies(
+      repositories = repositories,
+      depToDependency = depToDependency,
+      deps = deps,
+      sources = sources,
+      mapDependencies = mapDependencies,
+      customizer = None,
+      ctx = ctx
+    )
+
 }

scalalib/src/dependency/versions/VersionsFinder.scala

@@ -28,12 +28,20 @@ private[dependency] object VersionsFinder {
     javaModules.map { javaModule =>
       val depToDependency = eval(evaluator, javaModule.resolveCoursierDependency)
       val deps = evalOrElse(evaluator, javaModule.ivyDeps, Loose.Agg.empty[Dep])
+      val compileIvyDeps = evalOrElse(evaluator, javaModule.compileIvyDeps, Loose.Agg.empty[Dep])
+      val runIvyDeps = evalOrElse(evaluator, javaModule.runIvyDeps, Loose.Agg.empty[Dep])
       val repos = evalOrElse(evaluator, javaModule.repositoriesTask, Seq.empty[Repository])
+      val mapDeps = evalOrElse(evaluator, javaModule.mapDependencies, (d: coursier.Dependency) => d)
+      val custom = evalOrElse(evaluator, javaModule.resolutionCustomizer, None)
 
       val (dependencies, _) =
-        Lib.resolveDependenciesMetadata(repos,
-                                        depToDependency,
-                                        deps)
+        Lib.resolveDependenciesMetadata(
+          repositories = repos,
+          depToDependency = depToDependency,
+          deps = deps ++ compileIvyDeps ++ runIvyDeps,
+          mapDependencies = Some(mapDeps),
+          customizer = custom
+        )
 
       (javaModule, dependencies)
     }