From a204d22ffadcba137eee55116e6617cac1806fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wro=C5=84ski?= Date: Mon, 15 Nov 2021 12:33:24 +0100 Subject: [PATCH] Run native linker only if project was changed --- .../src/main/scala/scala/build/Inputs.scala | 19 +++ .../src/main/scala/scala/build/Sources.scala | 8 + .../scala/scala/cli/commands/Package.scala | 36 ++--- .../main/scala/scala/cli/commands/Run.scala | 9 +- .../cli/internal/NativeBuilderHelper.scala | 65 ++++++++ .../scala/cli/tests/internal/BuildTests.scala | 147 ++++++++++++++++++ .../internal/NativeBuilderHelperCheck.scala | 142 +++++++++++++++++ 7 files changed, 401 insertions(+), 25 deletions(-) create mode 100644 modules/cli/src/main/scala/scala/cli/internal/NativeBuilderHelper.scala create mode 100644 modules/cli/src/test/scala/cli/tests/internal/BuildTests.scala create mode 100644 modules/cli/src/test/scala/cli/tests/internal/NativeBuilderHelperCheck.scala diff --git a/modules/build/src/main/scala/scala/build/Inputs.scala b/modules/build/src/main/scala/scala/build/Inputs.scala index de95055e26..6abb41ae4d 100644 --- a/modules/build/src/main/scala/scala/build/Inputs.scala +++ b/modules/build/src/main/scala/scala/build/Inputs.scala @@ -124,6 +124,25 @@ final case class Inputs( if (canWrite) this else inHomeDir(directories) } + def sourceHash(): String = { + def bytes(s: String): Array[Byte] = s.getBytes(StandardCharsets.UTF_8) + val it = elements.iterator.flatMap { + case elem: Inputs.OnDisk => + val content = elem match { + case _: Inputs.Directory => "dir:" + case _: Inputs.ResourceDirectory => "resource-dir:" + case _ => os.read(elem.path) + } + Iterator(elem.path.toString, content, "\n").map(bytes) + case v: Inputs.Virtual => + Iterator(v.content, bytes("\n")) + } + val md = MessageDigest.getInstance("SHA-1") + it.foreach(md.update(_)) + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } } object Inputs { diff --git a/modules/build/src/main/scala/scala/build/Sources.scala b/modules/build/src/main/scala/scala/build/Sources.scala index 282d648c85..4eda556d96 100644 --- a/modules/build/src/main/scala/scala/build/Sources.scala +++ b/modules/build/src/main/scala/scala/build/Sources.scala @@ -35,6 +35,14 @@ final case class Sources( object Sources { + def empty: Sources = Sources( + paths = Seq.empty, + inMemory = Seq.empty, + mainClass = None, + resourceDirs = Seq.empty, + buildOptions = BuildOptions() + ) + def defaultPreprocessors(codeWrapper: CodeWrapper): Seq[Preprocessor] = Seq( ScriptPreprocessor(codeWrapper), diff --git a/modules/cli/src/main/scala/scala/cli/commands/Package.scala b/modules/cli/src/main/scala/scala/cli/commands/Package.scala index 34f70616ef..f943904a1e 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/Package.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/Package.scala @@ -1,13 +1,7 @@ package scala.cli.commands import caseapp._ -import coursier.launcher.{ - AssemblyGenerator, - BootstrapGenerator, - ClassPathEntry, - Parameters, - Preamble -} +import coursier.launcher._ import org.scalajs.linker.interface.StandardConfig import packager.config._ import packager.deb.DebianPackage @@ -30,7 +24,7 @@ import scala.build.internal.ScalaJsConfig import scala.build.options.{PackageType, Platform} import scala.build.{Build, Inputs, Logger, Os} import scala.cli.commands.OptionsHelper._ -import scala.cli.internal.{GetImageResizer, ScalaJsLinker} +import scala.cli.internal.{GetImageResizer, NativeBuilderHelper, ScalaJsLinker} import scala.scalanative.util.Scope import scala.scalanative.{build => sn} import scala.util.Properties @@ -549,18 +543,22 @@ object Package extends ScalaCommand[PackageOptions] { ): Unit = { os.makeDir.all(nativeWorkDir) + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, dest, nativeWorkDir) + + if (changed) + withLibraryJar(build, dest.last.stripSuffix(".jar")) { mainJar => + val config = sn.Config.empty + .withCompilerConfig(nativeConfig) + .withMainClass(mainClass + "$") + .withClassPath(mainJar +: build.artifacts.classPath) + .withWorkdir(nativeWorkDir.toNIO) + .withLogger(nativeLogger) + + Scope { implicit scope => + sn.Build.build(config, dest.toNIO) + } - withLibraryJar(build, dest.last.stripSuffix(".jar")) { mainJar => - val config = sn.Config.empty - .withCompilerConfig(nativeConfig) - .withMainClass(mainClass + "$") - .withClassPath(mainJar +: build.artifacts.classPath) - .withWorkdir(nativeWorkDir.toNIO) - .withLogger(nativeLogger) - - Scope { implicit scope => - sn.Build.build(config, dest.toNIO) + NativeBuilderHelper.updateOutputSha(dest, nativeWorkDir) } - } } } diff --git a/modules/cli/src/main/scala/scala/cli/commands/Run.scala b/modules/cli/src/main/scala/scala/cli/commands/Run.scala index 733b906e7c..424d2930bf 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/Run.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/Run.scala @@ -202,11 +202,8 @@ object Run extends ScalaCommand[RunOptions] { workDir: os.Path, logger: sn.Logger )(f: os.Path => T): T = { - val dest = os.temp(prefix = "main", suffix = if (Properties.isWin) ".exe" else "") - try { - Package.buildNative(build, mainClass, dest, config, workDir, logger) - f(dest) - } - finally if (os.exists(dest)) os.remove(dest) + val dest = workDir / s"main${if (Properties.isWin) ".exe" else ""}" + Package.buildNative(build, mainClass, dest, config, workDir, logger) + f(dest) } } diff --git a/modules/cli/src/main/scala/scala/cli/internal/NativeBuilderHelper.scala b/modules/cli/src/main/scala/scala/cli/internal/NativeBuilderHelper.scala new file mode 100644 index 0000000000..1ba9a94796 --- /dev/null +++ b/modules/cli/src/main/scala/scala/cli/internal/NativeBuilderHelper.scala @@ -0,0 +1,65 @@ +package scala.cli.internal + +import java.math.BigInteger +import java.security.MessageDigest + +import scala.build.Build +import scala.scalanative.{build => sn} + +case object NativeBuilderHelper { + + private def resolveProjectShaPath(nativeWorkDir: os.Path) = nativeWorkDir / ".project_sha" + private def resolveOutputShaPath(nativeWorkDir: os.Path) = nativeWorkDir / ".output_sha" + + private def fileSha(filePath: os.Path): String = { + val md = MessageDigest.getInstance("SHA-1") + md.update(os.read.bytes(filePath)) + + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } + + private def projectSha(build: Build.Successful, nativeConfig: sn.NativeConfig) = { + val md = MessageDigest.getInstance("SHA-1") + md.update(build.inputs.sourceHash().getBytes) + md.update(nativeConfig.toString.getBytes) + md.update(build.options.hash.getOrElse("").getBytes) + + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } + + def updateOutputSha(dest: os.Path, nativeWorkDir: os.Path) = { + val outputShaPath = resolveOutputShaPath(nativeWorkDir) + val sha = fileSha(dest) + os.write.over(outputShaPath, sha) + } + + def buildIfChanged( + build: Build.Successful, + nativeConfig: sn.NativeConfig, + dest: os.Path, + nativeWorkDir: os.Path + ): Boolean = { + val projectShaPath = resolveProjectShaPath(nativeWorkDir) + val outputShaPath = resolveOutputShaPath(nativeWorkDir) + + val currentProjectSha = projectSha(build, nativeConfig) + val currentOutputSha = if (os.exists(dest)) Some(fileSha(dest)) else None + + val previousProjectSha = if (os.exists(projectShaPath)) Some(os.read(projectShaPath)) else None + val previousOutputSha = if (os.exists(outputShaPath)) Some(os.read(outputShaPath)) else None + + val changed = + !previousProjectSha.contains(currentProjectSha) || + previousOutputSha != currentOutputSha || + !os.exists(dest) + + // update sha in .projectShaPath + if (changed) os.write.over(projectShaPath, currentProjectSha, createFolders = true) + + changed + } +} diff --git a/modules/cli/src/test/scala/cli/tests/internal/BuildTests.scala b/modules/cli/src/test/scala/cli/tests/internal/BuildTests.scala new file mode 100644 index 0000000000..36c381ce83 --- /dev/null +++ b/modules/cli/src/test/scala/cli/tests/internal/BuildTests.scala @@ -0,0 +1,147 @@ +package cli.tests.internal + +import dependency.{ScalaParameters, ScalaVersion} + +import java.io.IOException +import java.nio.charset.StandardCharsets +import java.security.SecureRandom +import java.util.concurrent.atomic.AtomicInteger + +import scala.build.internal.Constants +import scala.build.options.{BuildOptions, Scope} +import scala.build.{Artifacts, Build, Inputs, Project, ScalaCompiler, Sources} +import scala.util.control.NonFatal + +final case class BuildTests( + files: Seq[(os.RelPath, String)], + options: BuildOptions = BuildOptions() +) { + + def withClangPath(clangPath: os.Path) = copy( + options = options.copy( + scalaNativeOptions = options.scalaNativeOptions.copy( + clang = Some(clangPath.toString()) + ) + ) + ) + + def withClangppPath(clangppPath: os.Path) = copy( + options = options.copy( + scalaNativeOptions = options.scalaNativeOptions.copy( + clangpp = Some(clangppPath.toString()) + ) + ) + ) + + private def writeIn(dir: os.Path): Unit = + for ((relPath, content) <- files) { + val path = dir / relPath + os.write(path, content.getBytes(StandardCharsets.UTF_8), createFolders = true) + } + def root(): os.Path = { + val tmpDir = BuildTests.tmpDir + writeIn(tmpDir) + tmpDir + } + def fromRoot[T](f: (os.Path, Build.Successful) => T): T = + BuildTests.withTmpDir { tmpDir => + writeIn(tmpDir) + f(tmpDir, generateBuilds(tmpDir)) + } + + private def inputs(workDir: os.Path): Inputs = Inputs.empty(workDir).copy(elements = files.map { + case (relPath, _) => Inputs.ScalaFile(workDir, relPath.asSubPath) + }) + + private val emptyArtifacts: Artifacts = Artifacts( + compilerDependencies = Seq.empty, + compilerArtifacts = Seq.empty, + compilerPlugins = Seq.empty, + dependencies = Seq.empty, + detailedArtifacts = Seq.empty, + extraClassPath = Seq.empty, + extraCompileOnlyJars = Seq.empty, + extraSourceJars = Seq.empty, + params = ScalaParameters(BuildTests.defaultScalaVersion) + ) + + private def emptyProject(workDir: os.Path): Project = Project( + workspace = workDir, + classesDir = workDir, + scalaCompiler = ScalaCompiler( + BuildTests.defaultScalaVersion, + ScalaVersion.binary(BuildTests.defaultScalaVersion), + Seq.empty, + Seq.empty + ), + scalaJsOptions = None, + scalaNativeOptions = None, + projectName = "build-test", + classPath = Seq.empty, + sources = Seq.empty, + resolution = None, + resourceDirs = Seq.empty, + javaHomeOpt = None, + scope = Scope.Main, + javacOptions = Nil + ) + + private def generateBuilds(workDir: os.Path): Build.Successful = Build.Successful( + inputs = inputs(workDir), + options = options, + scalaParams = ScalaParameters(BuildTests.defaultScalaVersion), + scope = Scope.Main, + sources = Sources.empty, + artifacts = emptyArtifacts, + project = emptyProject(workDir), + output = workDir, + diagnostics = None + ) + +} + +object BuildTests { + + lazy val defaultScalaVersion = Constants.defaultScalaVersion + + private lazy val baseTmpDir = { + val base = os.temp.dir() + val rng = new SecureRandom + val d = base / s"run-${math.abs(rng.nextInt().toLong)}" + os.makeDir.all(d) + Runtime.getRuntime.addShutdownHook( + new Thread("scala-cli-its-clean-up-tmp-dir") { + setDaemon(true) + override def run(): Unit = + try os.remove.all(d) + catch { + case NonFatal(_) => + System.err.println(s"Could not remove $d, ignoring it.") + } + } + ) + d + } + + private val tmpCount = new AtomicInteger + + private def withTmpDir[T](f: os.Path => T): T = { + val tmpDir = baseTmpDir / s"test-${tmpCount.incrementAndGet()}" + os.makeDir.all(tmpDir) + val tmpDir0 = os.Path(tmpDir.toIO.getCanonicalFile) + def removeAll(): Unit = + try os.remove.all(tmpDir0) + catch { + case ex: IOException => + System.err.println(s"Ignoring $ex while removing $tmpDir0") + } + try f(tmpDir0) + finally removeAll() + } + + private def tmpDir: os.Path = { + val tmpDir = baseTmpDir / s"test-${tmpCount.incrementAndGet()}" + os.makeDir.all(tmpDir) + os.Path(tmpDir.toIO.getCanonicalFile) + } +} diff --git a/modules/cli/src/test/scala/cli/tests/internal/NativeBuilderHelperCheck.scala b/modules/cli/src/test/scala/cli/tests/internal/NativeBuilderHelperCheck.scala new file mode 100644 index 0000000000..2d17f0a3bc --- /dev/null +++ b/modules/cli/src/test/scala/cli/tests/internal/NativeBuilderHelperCheck.scala @@ -0,0 +1,142 @@ +package cli.tests.internal + +import com.eed3si9n.expecty.Expecty.{assert => expect} + +import scala.cli.internal.NativeBuilderHelper +import scala.util.{Properties, Random} + +class NativeBuilderHelperCheck extends munit.FunSuite { + + val helloFileName = "Hello.scala" + + val builds = BuildTests( + Seq( + os.rel / helloFileName -> + s"""object Hello extends App { + | println(msg) + |} + |""".stripMargin + ) + ).withClangPath(os.Path("dummy-clang-path", os.pwd)) + .withClangppPath(os.Path("dummy-clang-path", os.pwd)) + + test("should build native app at first time") { + + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changed) + } + } + + test("should not rebuild the second time") { + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + val changedSameBuild = + NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(!changedSameBuild) + } + } + + test("should build native if output file was deleted") { + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.remove(destPath) + val changedAfterDelete = + NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterDelete) + } + } + + test("should build native if output file was changed") { + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.write.over(destPath, Random.alphanumeric.take(10).mkString("")) + val changedAfterFileUpdate = + NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterFileUpdate) + } + } + + test("should build native if input file was changed") { + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.write.append(root / helloFileName, Random.alphanumeric.take(10).mkString("")) + val changedAfterFileUpdate = + NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterFileUpdate) + } + } + + test("should build native if native config was changed") { + builds.fromRoot { (root, build) => + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = NativeBuilderHelper.buildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + val updatedBuild = build.copy( + options = build.options.copy( + scalaNativeOptions = build.options.scalaNativeOptions.copy( + clang = Some(Random.alphanumeric.take(10).mkString("")) + ) + ) + ) + val updatedNativeConfig = updatedBuild.options.scalaNativeOptions.config + + val changedAfterConfigUpdate = + NativeBuilderHelper.buildIfChanged( + updatedBuild, + updatedNativeConfig, + destPath, + nativeWorkDir + ) + expect(changedAfterConfigUpdate) + } + } + +}