Allow for dependencies to fallback to a JAR url provided in a url user param

modules/directives/src/main/scala/scala/build/preprocessing/directives/UsingDependencyDirectiveHandler.scala

@@ -17,7 +17,8 @@ case object UsingDependencyDirectiveHandler extends UsingDirectiveHandler {
   override def usageMd = "`//> using lib \"`_org_`:`name`:`ver\""
   override def examples = Seq(
     "//> using lib \"org.scalatest::scalatest:3.2.10\"",
-    "//> using lib \"org.scalameta::munit:0.7.29\""
+    "//> using lib \"org.scalameta::munit:0.7.29\"",
+    "//> using lib \"tabby:tabby:0.2.3,url=https://github.com/bjornregnell/tabby/releases/download/v0.2.3/tabby_3-0.2.3.jar\""
   )
 
   private def parseDependency(depStr: String): Either[BuildException, AnyDependency] =

modules/options/src/main/scala/scala/build/Artifacts.scala

@@ -1,24 +1,26 @@
 package scala.build
 
 import coursier.cache.FileCache
-import coursier.core.Classifier
+import coursier.core.{Classifier, Module}
 import coursier.parse.RepositoryParser
 import coursier.util.Task
-import coursier.{Dependency => CsDependency, Fetch, core => csCore, util => csUtil}
-import dependency._
+import coursier.{Dependency as CsDependency, Fetch, core as csCore, util as csUtil}
+import dependency.*
 
-import scala.build.CoursierUtils._
+import java.net.URL
+
+import scala.build.CoursierUtils.*
 import scala.build.EitherCps.{either, value}
-import scala.build.Ops._
+import scala.build.Ops.*
 import scala.build.errors.{
   BuildException,
   CompositeBuildException,
   FetchingDependenciesError,
   RepositoryFormatError
 }
 import scala.build.internal.Constants
-import scala.build.internal.Constants._
-import scala.build.internal.CsLoggerUtil._
+import scala.build.internal.Constants.*
+import scala.build.internal.CsLoggerUtil.*
 import scala.build.internal.Util.ScalaDependencyOps
 
 final case class Artifacts(
@@ -154,11 +156,11 @@ object Artifacts {
       maybeSnapshotRepo ++ extraRepositories
 
     val internalDependencies =
-      jvmRunnerDependencies.map(Positioned.none(_)) ++
-        jvmTestRunnerDependencies.map(Positioned.none(_)) ++
-        jsTestBridgeDependencies.map(Positioned.none(_)) ++
-        nativeTestInterfaceDependencies.map(Positioned.none(_)) ++
-        jmhDependencies.map(Positioned.none(_))
+      jvmRunnerDependencies.map(Positioned.none) ++
+        jvmTestRunnerDependencies.map(Positioned.none) ++
+        jsTestBridgeDependencies.map(Positioned.none) ++
+        nativeTestInterfaceDependencies.map(Positioned.none) ++
+        jmhDependencies.map(Positioned.none)
     val updatedDependencies = dependencies ++ internalDependencies
 
     val updatedDependenciesMessage = {
@@ -223,11 +225,10 @@ object Artifacts {
         None
     }
 
-    val scalaNativeCli = fetchedScalaNativeCli.toSeq.flatMap { fetched =>
-      fetched.fullDetailedArtifacts.collect { case (_, _, _, Some(f)) =>
-        os.Path(f, Os.pwd)
-      }
-    }
+    def fetchedArtifactToPath(fetched: Fetch.Result): Seq[os.Path] =
+      fetched.fullDetailedArtifacts.collect { case (_, _, _, Some(f)) => os.Path(f, Os.pwd) }
+
+    val scalaNativeCli = fetchedScalaNativeCli.toSeq.flatMap(fetchedArtifactToPath)
 
     val fetchedScalaJsCli = scalaJsCliDependency match {
       case Some(dependency) =>
@@ -251,11 +252,7 @@ object Artifacts {
         None
     }
 
-    val scalaJsCli = fetchedScalaJsCli.toSeq.flatMap { fetched =>
-      fetched.fullDetailedArtifacts.collect { case (_, _, _, Some(f)) =>
-        os.Path(f, Os.pwd)
-      }
-    }
+    val scalaJsCli = fetchedScalaJsCli.toSeq.flatMap(fetchedArtifactToPath)
 
     val extraStubsJars =
       if (addStubs)
@@ -351,16 +348,32 @@ object Artifacts {
     logger: Logger,
     cache: FileCache[Task],
     classifiersOpt: Option[Set[String]]
-  ): Either[BuildException, Fetch.Result] =
+  ): Either[BuildException, Fetch.Result] = {
+    val coursierDependenciesWithFallbacks
+      : Positioned[Seq[(CsDependency, Option[((Module, String), (URL, Boolean))])]] =
+      dependencies.map(positioned =>
+        for {
+          dep <- positioned
+          csDep    = dep.toCs(params)
+          maybeUrl = dep.userParams.get("url").flatten.map(new URL(_))
+          fallback = maybeUrl.map(url => (csDep.module -> csDep.version) -> (url -> true))
+        } yield csDep -> fallback
+      )
+    val coursierDependencies: Positioned[Seq[CsDependency]] =
+      coursierDependenciesWithFallbacks.map(_.map(_._1))
+    val fallbacks: Map[(Module, String), (URL, Boolean)] =
+      coursierDependenciesWithFallbacks.value.flatMap(_._2).toMap
     fetch0(
-      dependencies.map(_.map(_.toCs(params))),
+      coursierDependencies,
       extraRepositories,
       Some(params.scalaVersion),
       Nil,
       logger,
       cache,
-      classifiersOpt
+      classifiersOpt,
+      fallbacks
     )
+  }
 
   def fetch0(
     dependencies: Positioned[Seq[coursier.Dependency]],
@@ -369,19 +382,24 @@ object Artifacts {
     forcedVersions: Seq[(coursier.Module, String)],
     logger: Logger,
     cache: FileCache[Task],
-    classifiersOpt: Option[Set[String]]
+    classifiersOpt: Option[Set[String]],
+    fallbacks: Map[(Module, String), (URL, Boolean)] = Map.empty
   ): Either[BuildException, Fetch.Result] = either {
     logger.debug {
       s"Fetching ${dependencies.value}" +
         (if (extraRepositories.isEmpty) "" else s", adding $extraRepositories")
     }
 
+    val fallbackRepository = TemporaryInMemoryRepository(fallbacks)
+
     val extraRepositories0 = value {
       RepositoryParser.repositories(extraRepositories)
         .either
         .left.map(errors => new RepositoryFormatError(errors))
     }
 
+    val extraRepositoriesWithFallback = extraRepositories0 :+ fallbackRepository
+
     val forceScalaVersions = forceScalaVersionOpt match {
       case None => Nil
       case Some(sv) =>
@@ -403,20 +421,19 @@ object Artifacts {
           )
     }
 
+    val forceVersion = forceScalaVersions ++ forcedVersions
+
     // FIXME Many parameters that we could allow to customize here
     var fetcher = coursier.Fetch()
       .withCache(cache)
-      .addRepositories(extraRepositories0: _*)
-      .addDependencies(dependencies.value: _*)
-      .mapResolutionParams { params =>
-        params
-          .addForceVersion(forceScalaVersions ++ forcedVersions: _*)
-      }
+      .addRepositories(extraRepositoriesWithFallback*)
+      .addDependencies(dependencies.value*)
+      .mapResolutionParams(_.addForceVersion(forceVersion*))
     for (classifiers <- classifiersOpt) {
       if (classifiers("_"))
         fetcher = fetcher.withMainArtifacts()
       fetcher = fetcher
-        .addClassifiers(classifiers.toSeq.filter(_ != "_").map(coursier.Classifier(_)): _*)
+        .addClassifiers(classifiers.toSeq.filter(_ != "_").map(coursier.Classifier(_))*)
     }
 
     val res = cache.logger.use {

modules/options/src/main/scala/scala/build/TemporaryInMemoryRepository.scala

@@ -0,0 +1,212 @@
+package scala.build
+
+import coursier.cache.{ConnectionBuilder, FileCache}
+import coursier.core.*
+import coursier.util.{Artifact, EitherT, Monad}
+
+import java.io.{File, FileNotFoundException, IOException}
+import java.net.{HttpURLConnection, URL, URLConnection}
+
+import scala.util.Try
+
+/** Copied over from [[https://github.com/coursier/sbt-coursier]]. Should probably be removed in the
+  * future, when (and if) the source lands in coursier itself.
+  *
+  * [[https://github.com/coursier/sbt-coursier/blob/master/modules/lm-coursier/src/main/scala/lmcoursier/internal/TemporaryInMemoryRepository.scala]]
+  */
+object TemporaryInMemoryRepository {
+
+  def closeConn(conn: URLConnection): Unit = {
+    Try(conn.getInputStream).toOption.filter(_ != null).foreach(_.close())
+    conn match {
+      case conn0: HttpURLConnection =>
+        Try(conn0.getErrorStream).toOption.filter(_ != null).foreach(_.close())
+        conn0.disconnect()
+      case _ =>
+    }
+  }
+
+  def exists(
+    url: URL,
+    localArtifactsShouldBeCached: Boolean
+  ): Boolean =
+    exists(url, localArtifactsShouldBeCached, None)
+
+  def exists(
+    url: URL,
+    localArtifactsShouldBeCached: Boolean,
+    cacheOpt: Option[FileCache[Nothing]]
+  ): Boolean = {
+
+    // Sometimes HEAD attempts fail even though standard GETs are fine.
+    // E.g. https://github.com/NetLogo/NetLogo/releases/download/5.3.1/NetLogo.jar
+    // returning 403s. Hence the second attempt below.
+
+    val protocolSpecificAttemptOpt = {
+
+      def ifFile: Option[Boolean] =
+        if (localArtifactsShouldBeCached && !new File(url.toURI).exists()) {
+          val cachePath = coursier.cache.CacheDefaults.location
+          // 'file' here stands for the protocol (e.g. it's https instead for https:// URLs)
+          Some(new File(cachePath, s"file/${url.getPath}").exists())
+        }
+        else
+          Some(new File(url.toURI).exists()) // FIXME Escaping / de-escaping needed here?
+
+      def ifHttp: Option[Boolean] = {
+        // HEAD request attempt, adapted from http://stackoverflow.com/questions/22541629/android-how-can-i-make-an-http-head-request/22545275#22545275
+
+        var conn: URLConnection = null
+        try {
+          conn = ConnectionBuilder(url.toURI.toASCIIString)
+            .withFollowHttpToHttpsRedirections(
+              cacheOpt.fold(false)(_.followHttpToHttpsRedirections)
+            )
+            .withFollowHttpsToHttpRedirections(
+              cacheOpt.fold(false)(_.followHttpsToHttpRedirections)
+            )
+            .withSslSocketFactoryOpt(cacheOpt.flatMap(_.sslSocketFactoryOpt))
+            .withHostnameVerifierOpt(cacheOpt.flatMap(_.hostnameVerifierOpt))
+            .withMethod("HEAD")
+            .withMaxRedirectionsOpt(cacheOpt.flatMap(_.maxRedirections))
+            .connection()
+          // Even though the finally clause handles this too, this has to be run here, so that we return Some(true)
+          // iff this doesn't throw.
+          conn.getInputStream.close()
+          Some(true)
+        }
+        catch {
+          case _: FileNotFoundException => Some(false)
+          case _: IOException           => None // error other than not found
+        }
+        finally
+          if (conn != null)
+            closeConn(conn)
+      }
+
+      url.getProtocol match {
+        case "file"           => ifFile
+        case "http" | "https" => ifHttp
+        case _                => None
+      }
+    }
+
+    def genericAttempt: Boolean = {
+      var conn: URLConnection = null
+      try {
+        conn = url.openConnection()
+        // NOT setting request type to HEAD here.
+        conn.getInputStream.close()
+        true
+      }
+      catch {
+        case _: IOException => false
+      }
+      finally
+        if (conn != null)
+          closeConn(conn)
+    }
+
+    protocolSpecificAttemptOpt
+      .getOrElse(genericAttempt)
+  }
+
+  def apply(fallbacks: Map[(Module, String), (URL, Boolean)]): TemporaryInMemoryRepository =
+    new TemporaryInMemoryRepository(fallbacks, localArtifactsShouldBeCached = false, None)
+
+  def apply(
+    fallbacks: Map[(Module, String), (URL, Boolean)],
+    localArtifactsShouldBeCached: Boolean
+  ): TemporaryInMemoryRepository =
+    new TemporaryInMemoryRepository(fallbacks, localArtifactsShouldBeCached, None)
+
+  def apply[F[_]](
+    fallbacks: Map[(Module, String), (URL, Boolean)],
+    cache: FileCache[F]
+  ): TemporaryInMemoryRepository =
+    new TemporaryInMemoryRepository(
+      fallbacks,
+      localArtifactsShouldBeCached = cache.localArtifactsShouldBeCached,
+      Some(cache.asInstanceOf[FileCache[Nothing]])
+    )
+
+}
+
+final class TemporaryInMemoryRepository private (
+  val fallbacks: Map[(Module, String), (URL, Boolean)],
+  val localArtifactsShouldBeCached: Boolean,
+  val cacheOpt: Option[FileCache[Nothing]]
+) extends Repository {
+
+  def find[F[_]](
+    module: Module,
+    version: String,
+    fetch: Repository.Fetch[F]
+  )(implicit
+    F: Monad[F]
+  ): EitherT[F, String, (ArtifactSource, Project)] = {
+
+    def res = fallbacks
+      .get((module, version))
+      .fold[Either[String, (ArtifactSource, Project)]](Left("No fallback URL found")) {
+        case (url, _) =>
+          val urlStr = url.toExternalForm
+          val idx    = urlStr.lastIndexOf('/')
+
+          if (idx < 0 || urlStr.endsWith("/"))
+            Left(s"$url doesn't point to a file")
+          else {
+            val (dirUrlStr, fileName) = urlStr.splitAt(idx + 1)
+
+            if (TemporaryInMemoryRepository.exists(url, localArtifactsShouldBeCached, cacheOpt)) {
+              val proj = Project(
+                module,
+                version,
+                Nil,
+                Map.empty,
+                None,
+                Nil,
+                Nil,
+                Nil,
+                None,
+                None,
+                None,
+                relocated = false,
+                None,
+                Nil,
+                Info.empty
+              )
+
+              Right((this, proj))
+            }
+            else
+              Left(s"$fileName not found under $dirUrlStr")
+          }
+      }
+
+    // EitherT(F.bind(F.point(()))(_ => F.point(res)))
+    EitherT(F.map(F.point(()))(_ => res))
+  }
+
+  def artifacts(
+    dependency: Dependency,
+    project: Project,
+    overrideClassifiers: Option[Seq[Classifier]]
+  ): Seq[(Publication, Artifact)] =
+    fallbacks
+      .get(dependency.moduleVersion)
+      .toSeq
+      .map {
+        case (url, changing) =>
+          val url0 = url.toString
+          val ext  = url0.substring(url0.lastIndexOf('.') + 1)
+          val pub = Publication(
+            dependency.module.name.value, // ???
+            Type(ext),
+            Extension(ext),
+            Classifier.empty
+          )
+          (pub, Artifact(url0, Map.empty, Map.empty, changing, optional = false, None))
+      }
+
+}