Support building Graal native images in docker (#1251)

Closes #1250

This provides support for building Graal native images in a docker
container.

.travis.yml

@@ -62,6 +62,7 @@ jobs:
         gu install native-image
         sbt "^validateGraalVMNativeImage"
       if: type = pull_request OR (type = push AND branch = master)
+      services: docker
       name: "scripted GraalVM native-image tests"
     - script: sbt "^validateJar"
       name: "scripted jar tests"

build.sbt

@@ -68,7 +68,19 @@ mimaBinaryIssueFilters ++= {
     ProblemFilters.exclude[MissingTypesProblem]("com.typesafe.sbt.packager.rpm.RpmMetadata$"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("com.typesafe.sbt.packager.rpm.RpmMetadata.apply"),
     ProblemFilters.exclude[DirectMissingMethodProblem]("com.typesafe.sbt.packager.rpm.RpmMetadata.copy"),
-    ProblemFilters.exclude[DirectMissingMethodProblem]("com.typesafe.sbt.packager.rpm.RpmMetadata.this")
+    ProblemFilters.exclude[DirectMissingMethodProblem]("com.typesafe.sbt.packager.rpm.RpmMetadata.this"),
+    // added via #1251
+    ProblemFilters.exclude[ReversedMissingMethodProblem](
+      "com.typesafe.sbt.packager.universal.UniversalKeys.com$typesafe$sbt$packager$universal$UniversalKeys$_setter_$containerBuildImage_="
+    ),
+    ProblemFilters
+      .exclude[ReversedMissingMethodProblem]("com.typesafe.sbt.packager.universal.UniversalKeys.containerBuildImage"),
+    ProblemFilters.exclude[ReversedMissingMethodProblem](
+      "com.typesafe.sbt.packager.graalvmnativeimage.GraalVMNativeImageKeys.graalVMNativeImageGraalVersion"
+    ),
+    ProblemFilters.exclude[ReversedMissingMethodProblem](
+      "com.typesafe.sbt.packager.graalvmnativeimage.GraalVMNativeImageKeys.com$typesafe$sbt$packager$graalvmnativeimage$GraalVMNativeImageKeys$_setter_$graalVMNativeImageGraalVersion_="
+    )
   )
 }
 

src/main/scala-sbt-0.13/com/typesafe/sbt/packager/MappingsHelper.scala

@@ -111,4 +111,11 @@ object MappingsHelper {
         file -> s"$target/${file.getName}"
     }
 
+  /**
+    * Get the mappings for the given files relative to the given directories.
+    */
+  def relative(files: Seq[File], dirs: Seq[File]): Seq[(File, String)] = {
+    (files --- dirs) pair (relativeTo(dirs) | Path.flat)
+  }
+
 }

src/main/scala-sbt-1.0/com/typesafe/sbt/packager/MappingsHelper.scala

@@ -81,4 +81,10 @@ object MappingsHelper extends Mapper {
 	file -> s"$target/${file.getName}"
     }
 
+  /**
+    * Get the mappings for the given files relative to the given directories.
+    */
+  def relative(files: Seq[File], dirs: Seq[File]): Seq[(File, String)] = {
+    (files --- dirs) pair (relativeTo(dirs) | flat)
+  }
 }

src/main/scala/com/typesafe/sbt/packager/docker/DockerPlugin.scala

@@ -106,7 +106,6 @@ object DockerPlugin extends AutoPlugin {
     },
     dockerEntrypoint := Seq(s"${(defaultLinuxInstallLocation in Docker).value}/bin/${executableScriptName.value}"),
     dockerCmd := Seq(),
-    dockerExecCommand := Seq("docker"),
     dockerVersion := Try(Process(dockerExecCommand.value ++ Seq("version", "--format", "'{{.Server.Version}}'")).!!).toOption
       .map(_.trim)
       .flatMap(DockerVersion.parse),
@@ -478,7 +477,7 @@ object DockerPlugin extends AutoPlugin {
     inConfig(Docker)(Seq(mappings := renameDests((mappings in Universal).value, defaultLinuxInstallLocation.value)))
   }
 
-  private[docker] def publishLocalLogger(log: Logger) =
+  private[packager] def publishLocalLogger(log: Logger) =
     new sys.process.ProcessLogger {
       override def err(err: => String): Unit =
         err match {

src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImageKeys.scala

@@ -0,0 +1,17 @@
+package com.typesafe.sbt
+package packager
+package graalvmnativeimage
+
+import sbt._
+
+/**
+  * GraalVM settings
+  */
+trait GraalVMNativeImageKeys {
+  val graalVMNativeImageOptions =
+    settingKey[Seq[String]]("GraalVM native-image options")
+
+  val graalVMNativeImageGraalVersion = settingKey[Option[String]](
+    "Version of GraalVM to build with. Setting this has the effect of generating a container build image to build the native image with this version of GraalVM."
+  )
+}

src/main/scala/com/typesafe/sbt/packager/graalvmnativeimage/GraalVMNativeImagePlugin.scala

@@ -0,0 +1,193 @@
+package com.typesafe.sbt.packager.graalvmnativeimage
+
+import java.io.ByteArrayInputStream
+
+import sbt._
+import sbt.Keys.{mainClass, name, _}
+import com.typesafe.sbt.packager.{MappingsHelper, Stager}
+import com.typesafe.sbt.packager.Keys._
+import com.typesafe.sbt.packager.Compat._
+import com.typesafe.sbt.packager.archetypes.JavaAppPackaging
+import com.typesafe.sbt.packager.docker.{Cmd, DockerPlugin, Dockerfile, ExecCmd}
+import com.typesafe.sbt.packager.universal.UniversalPlugin
+
+/**
+  * Plugin to compile ahead-of-time native executables.
+  *
+  * @example Enable the plugin in the `build.sbt`
+  * {{{
+  *    enablePlugins(GraalVMNativeImagePlugin)
+  * }}}
+  */
+object GraalVMNativeImagePlugin extends AutoPlugin {
+
+  object autoImport extends GraalVMNativeImageKeys {
+    val GraalVMNativeImage: Configuration = config("graalvm-native-image")
+  }
+
+  import autoImport._
+
+  private val GraalVMBaseImage = "oracle/graalvm-ce"
+  private val NativeImageCommand = "native-image"
+
+  override def requires: Plugins = JavaAppPackaging
+
+  override def projectConfigurations: Seq[Configuration] = Seq(GraalVMNativeImage)
+
+  override lazy val projectSettings: Seq[Setting[_]] = Seq(
+    target in GraalVMNativeImage := target.value / "graalvm-native-image",
+    graalVMNativeImageOptions := Seq.empty,
+    graalVMNativeImageGraalVersion := None,
+    resourceDirectory in GraalVMNativeImage := sourceDirectory.value / "graal",
+    mainClass in GraalVMNativeImage := (mainClass in Compile).value
+  ) ++ inConfig(GraalVMNativeImage)(scopedSettings)
+
+  private lazy val scopedSettings = Seq[Setting[_]](
+    resourceDirectories := Seq(resourceDirectory.value),
+    includeFilter := "*",
+    resources := resourceDirectories.value.descendantsExcept(includeFilter.value, excludeFilter.value).get,
+    UniversalPlugin.autoImport.containerBuildImage := Def.taskDyn {
+      graalVMNativeImageGraalVersion.value match {
+        case Some(tag) => generateContainerBuildImage(s"$GraalVMBaseImage:$tag")
+        case None      => Def.task(None: Option[String])
+      }
+    }.value,
+    packageBin := {
+      val targetDirectory = target.value
+      val binaryName = name.value
+      val className = mainClass.value.getOrElse(sys.error("Could not find a main class."))
+      val classpathJars = scriptClasspathOrdering.value
+      val extraOptions = graalVMNativeImageOptions.value
+      val streams = Keys.streams.value
+      val dockerCommand = DockerPlugin.autoImport.dockerExecCommand.value
+      val graalResourceDirectories = resourceDirectories.value
+      val graalResources = resources.value
+
+      UniversalPlugin.autoImport.containerBuildImage.value match {
+        case None =>
+          buildLocal(targetDirectory, binaryName, className, classpathJars.map(_._1), extraOptions, streams.log)
+
+        case Some(image) =>
+          val resourceMappings = MappingsHelper.relative(graalResources, graalResourceDirectories)
+
+          buildInDockerContainer(
+            targetDirectory,
+            binaryName,
+            className,
+            classpathJars,
+            extraOptions,
+            dockerCommand,
+            resourceMappings,
+            image,
+            streams
+          )
+      }
+    }
+  )
+
+  private def buildLocal(targetDirectory: File,
+                         binaryName: String,
+                         className: String,
+                         classpathJars: Seq[File],
+                         extraOptions: Seq[String],
+                         log: ProcessLogger): File = {
+    targetDirectory.mkdirs()
+    val command = {
+      val nativeImageArguments = {
+        val classpath = classpathJars.mkString(":")
+        Seq("--class-path", classpath, s"-H:Name=$binaryName") ++ extraOptions ++ Seq(className)
+      }
+      Seq(NativeImageCommand) ++ nativeImageArguments
+    }
+    sys.process.Process(command, targetDirectory) ! log match {
+      case 0 => targetDirectory / binaryName
+      case x => sys.error(s"Failed to run $command, exit status: " + x)
+    }
+  }
+
+  private def buildInDockerContainer(targetDirectory: File,
+                                     binaryName: String,
+                                     className: String,
+                                     classpathJars: Seq[(File, String)],
+                                     extraOptions: Seq[String],
+                                     dockerCommand: Seq[String],
+                                     resources: Seq[(File, String)],
+                                     image: String,
+                                     streams: TaskStreams): File = {
+
+    stage(targetDirectory, classpathJars, resources, streams)
+
+    val command = dockerCommand ++ Seq(
+      "run",
+      "--rm",
+      "-v",
+      s"${targetDirectory.getAbsolutePath}:/opt/graalvm",
+      image,
+      "-cp",
+      classpathJars.map(jar => "/opt/graalvm/stage/" + jar._2).mkString(":"),
+      s"-H:Name=$binaryName"
+    ) ++ extraOptions ++ Seq(className)
+
+    sys.process.Process(command) ! streams.log match {
+      case 0 => targetDirectory / binaryName
+      case x => sys.error(s"Failed to run $command, exit status: " + x)
+    }
+  }
+
+  /**
+    * This can be used to build a custom build image starting from a custom base image. Can be used like so:
+    *
+    * ```
+    * (containerBuildImage in GraalVMNativeImage) := generateContainerBuildImage("my-docker-hub-username/my-graalvm").value
+    * ```
+    *
+    * The passed in docker image must have GraalVM installed and on the PATH, including the gu utility.
+    */
+  def generateContainerBuildImage(baseImage: String): Def.Initialize[Task[Option[String]]] = Def.task {
+    val dockerCommand = (DockerPlugin.autoImport.dockerExecCommand in GraalVMNativeImage).value
+    val streams = Keys.streams.value
+
+    val (baseName, tag) = baseImage.split(":", 2) match {
+      case Array(n, t) => (n, t)
+      case Array(n)    => (n, "latest")
+    }
+
+    val imageName = s"${baseName.replace('/', '-')}-native-image:$tag"
+    import sys.process._
+    if ((dockerCommand ++ Seq("image", "ls", imageName, "--quiet")).!!.trim.isEmpty) {
+      streams.log.info(s"Generating new GraalVM native-image image based on $baseImage: $imageName")
+
+      val dockerContent = Dockerfile(
+        Cmd("FROM", baseImage),
+        Cmd("WORKDIR", "/opt/graalvm"),
+        ExecCmd("RUN", "gu", "install", "native-image"),
+        ExecCmd("ENTRYPOINT", "native-image")
+      ).makeContent
+
+      val command = dockerCommand ++ Seq("build", "-t", imageName, "-")
+
+      val ret = sys.process.Process(command) #<
+        new ByteArrayInputStream(dockerContent.getBytes()) !
+        DockerPlugin.publishLocalLogger(streams.log)
+
+      if (ret != 0)
+        throw new RuntimeException("Nonzero exit value when generating GraalVM container build image: " + ret)
+
+    } else {
+      streams.log.info(s"Using existing GraalVM native-image image: $imageName")
+    }
+
+    Some(imageName)
+  }
+
+  private def stage(targetDirectory: File,
+                    classpathJars: Seq[(File, String)],
+                    resources: Seq[(File, String)],
+                    streams: TaskStreams): File = {
+    val stageDir = targetDirectory / "stage"
+    val mappings = classpathJars ++ resources.map {
+      case (resource, path) => resource -> s"resources/$path"
+    }
+    Stager.stage(GraalVMBaseImage)(streams, stageDir, mappings)
+  }
+}

src/main/scala/com/typesafe/sbt/packager/universal/Keys.scala

@@ -19,4 +19,8 @@ trait UniversalKeys {
   val topLevelDirectory = SettingKey[Option[String]]("topLevelDirectory", "Top level dir in compressed output file.")
   val universalArchiveOptions =
     SettingKey[Seq[String]]("universal-archive-options", "Options passed to the tar/zip command. Scope by task")
+
+  val containerBuildImage = taskKey[Option[String]](
+    "For plugins that support building artifacts inside a docker container, if this is defined, this image will be used to do the building."
+  )
 }

src/main/scala/com/typesafe/sbt/packager/universal/UniversalPlugin.scala

@@ -5,6 +5,7 @@ import sbt.Keys._
 import Archives._
 import com.typesafe.sbt.SbtNativePackager
 import com.typesafe.sbt.packager.Keys._
+import com.typesafe.sbt.packager.docker.DockerPlugin
 import com.typesafe.sbt.packager.validation._
 import com.typesafe.sbt.packager.{SettingsHelper, Stager}
 import sbt.Keys.TaskStreams
@@ -46,6 +47,14 @@ object UniversalPlugin extends AutoPlugin {
   override def projectConfigurations: Seq[Configuration] =
     Seq(Universal, UniversalDocs, UniversalSrc)
 
+  override lazy val buildSettings: Seq[Setting[_]] = Seq[Setting[_]](
+    // Since more than just the docker plugin uses the docker command, we define this in the universal plugin
+    // so that it can be configured once and shared by all plugins without requiring the docker plugin. Also, make it
+    // a build settings so that it can be overridden once, at the build level.
+    DockerPlugin.autoImport.dockerExecCommand := Seq("docker"),
+    containerBuildImage := None
+  )
+
   /** The basic settings for the various packaging types. */
   override lazy val projectSettings: Seq[Setting[_]] = Seq[Setting[_]](
     // For now, we provide delegates from dist/stage to universal...

src/sbt-test/graalvm-native-image/docker-native-image/build.sbt

@@ -0,0 +1,5 @@
+enablePlugins(GraalVMNativeImagePlugin)
+
+name := "docker-test"
+version := "0.1.0"
+graalVMNativeImageGraalVersion := Some("19.0.0")

src/sbt-test/graalvm-native-image/docker-native-image/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))

src/sbt-test/graalvm-native-image/docker-native-image/src/main/scala/Main.scala

@@ -0,0 +1,5 @@
+object Main {
+  def main(args: Array[String]): Unit = {
+    println("Hello Graal")
+  }
+}