From 2f95997591ac20f962a32bc3e230acf609c9a051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Wro=C5=84ski?= Date: Mon, 15 Nov 2021 12:33:24 +0100 Subject: [PATCH] Run native linker only if project was changed --- .../src/main/scala/scala/build/Inputs.scala | 19 ++ .../build/internal/NativeBuilderHelper.scala | 65 +++++++ .../tests/NativeBuilderHelperTests.scala | 178 ++++++++++++++++++ .../scala/scala/cli/commands/Package.scala | 36 ++-- .../main/scala/scala/cli/commands/Run.scala | 9 +- 5 files changed, 282 insertions(+), 25 deletions(-) create mode 100644 modules/build/src/main/scala/scala/build/internal/NativeBuilderHelper.scala create mode 100644 modules/build/src/test/scala/scala/build/tests/NativeBuilderHelperTests.scala diff --git a/modules/build/src/main/scala/scala/build/Inputs.scala b/modules/build/src/main/scala/scala/build/Inputs.scala index de95055e26..6abb41ae4d 100644 --- a/modules/build/src/main/scala/scala/build/Inputs.scala +++ b/modules/build/src/main/scala/scala/build/Inputs.scala @@ -124,6 +124,25 @@ final case class Inputs( if (canWrite) this else inHomeDir(directories) } + def sourceHash(): String = { + def bytes(s: String): Array[Byte] = s.getBytes(StandardCharsets.UTF_8) + val it = elements.iterator.flatMap { + case elem: Inputs.OnDisk => + val content = elem match { + case _: Inputs.Directory => "dir:" + case _: Inputs.ResourceDirectory => "resource-dir:" + case _ => os.read(elem.path) + } + Iterator(elem.path.toString, content, "\n").map(bytes) + case v: Inputs.Virtual => + Iterator(v.content, bytes("\n")) + } + val md = MessageDigest.getInstance("SHA-1") + it.foreach(md.update(_)) + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } } object Inputs { diff --git a/modules/build/src/main/scala/scala/build/internal/NativeBuilderHelper.scala b/modules/build/src/main/scala/scala/build/internal/NativeBuilderHelper.scala new file mode 100644 index 0000000000..20f7a5f7c7 --- /dev/null +++ b/modules/build/src/main/scala/scala/build/internal/NativeBuilderHelper.scala @@ -0,0 +1,65 @@ +package scala.build.internal + +import java.math.BigInteger +import java.security.MessageDigest + +import scala.build.Build +import scala.scalanative.{build => sn} + +case object NativeBuilderHelper { + + private def resolveProjectShaPath(nativeWorkDir: os.Path) = nativeWorkDir / ".project_sha" + private def resolveOutputShaPath(nativeWorkDir: os.Path) = nativeWorkDir / ".output_sha" + + private def fileSha(filePath: os.Path): String = { + val md = MessageDigest.getInstance("SHA-1") + md.update(os.read.bytes(filePath)) + + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } + + private def projectSha(build: Build.Successful, nativeConfig: sn.NativeConfig) = { + val md = MessageDigest.getInstance("SHA-1") + md.update(build.inputs.sourceHash().getBytes) + md.update(nativeConfig.toString.getBytes) + md.update(build.options.hash.getOrElse("").getBytes) + + val digest = md.digest() + val calculatedSum = new BigInteger(1, digest) + String.format(s"%040x", calculatedSum) + } + + def updateOutputSha(dest: os.Path, nativeWorkDir: os.Path) = { + val outputShaPath = resolveOutputShaPath(nativeWorkDir) + val sha = fileSha(dest) + os.write.over(outputShaPath, sha) + } + + def shouldBuildIfChanged( + build: Build.Successful, + nativeConfig: sn.NativeConfig, + dest: os.Path, + nativeWorkDir: os.Path + ): Boolean = { + val projectShaPath = resolveProjectShaPath(nativeWorkDir) + val outputShaPath = resolveOutputShaPath(nativeWorkDir) + + val currentProjectSha = projectSha(build, nativeConfig) + val currentOutputSha = if (os.exists(dest)) Some(fileSha(dest)) else None + + val previousProjectSha = if (os.exists(projectShaPath)) Some(os.read(projectShaPath)) else None + val previousOutputSha = if (os.exists(outputShaPath)) Some(os.read(outputShaPath)) else None + + val changed = + !previousProjectSha.contains(currentProjectSha) || + previousOutputSha != currentOutputSha || + !os.exists(dest) + + // update sha in .projectShaPath + if (changed) os.write.over(projectShaPath, currentProjectSha, createFolders = true) + + changed + } +} diff --git a/modules/build/src/test/scala/scala/build/tests/NativeBuilderHelperTests.scala b/modules/build/src/test/scala/scala/build/tests/NativeBuilderHelperTests.scala new file mode 100644 index 0000000000..fb3e5b6549 --- /dev/null +++ b/modules/build/src/test/scala/scala/build/tests/NativeBuilderHelperTests.scala @@ -0,0 +1,178 @@ +package scala.build.tests + +import com.eed3si9n.expecty.Expecty.{assert => expect} + +import scala.build.Ops.EitherThrowOps +import scala.build.blooprifle.BloopRifleConfig +import scala.build.internal.NativeBuilderHelper +import scala.build.{Bloop, BuildThreads, Directories, LocalRepo, Logger} +import scala.build.options.{BuildOptions, InternalOptions, ScalaOptions} +import scala.util.{Properties, Random} + +class NativeBuilderHelperTests extends munit.FunSuite { + + val buildThreads = BuildThreads.create() + val bloopConfig = BloopRifleConfig.default(v => Bloop.bloopClassPath(Logger.nop, v)) + + val helloFileName = "Hello.scala" + + val inputs = TestInputs( + os.rel / helloFileName -> + s"""object Hello extends App { + | println("Hello") + |} + |""".stripMargin, + os.rel / "main" / "Main.scala" -> + s"""object Main extends App { + | println("Hello") + |} + |""".stripMargin + ) + + val extraRepoTmpDir = os.temp.dir(prefix = "scala-cli-tests-extra-repo-") + val directories = Directories.under(extraRepoTmpDir) + + val defaultOptions = BuildOptions( + internal = InternalOptions( + localRepository = LocalRepo.localRepo(directories.localRepoDir) + ) + ) + + test("should build native app at first time") { + + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changed) + } + } + + test("should not rebuild the second time") { + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + val changedSameBuild = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(!changedSameBuild) + } + } + + test("should build native if output file was deleted") { + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.remove(destPath) + val changedAfterDelete = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterDelete) + } + } + + test("should build native if output file was changed") { + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + // generate dummy output + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.write.over(destPath, Random.alphanumeric.take(10).mkString("")) + val changedAfterFileUpdate = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterFileUpdate) + } + } + + test("should build native if input file was changed") { + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + os.write.append(root / helloFileName, Random.alphanumeric.take(10).mkString("")) + val changedAfterFileUpdate = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + expect(changedAfterFileUpdate) + } + } + + test("should build native if native config was changed") { + inputs.withBuild(defaultOptions, buildThreads, bloopConfig) { (root, _, maybeBuild) => + val build = maybeBuild.toOption.get.successfulOpt.get + + val nativeConfig = build.options.scalaNativeOptions.config + val nativeWorkDir = build.options.scalaNativeOptions.nativeWorkDir(root, "native-test") + val destPath = nativeWorkDir / s"main${if (Properties.isWin) ".exe" else ""}" + os.write(destPath, Random.alphanumeric.take(10).mkString(""), createFolders = true) + + val changed = + NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, destPath, nativeWorkDir) + NativeBuilderHelper.updateOutputSha(destPath, nativeWorkDir) + expect(changed) + + val updatedBuild = build.copy( + options = build.options.copy( + scalaNativeOptions = build.options.scalaNativeOptions.copy( + clang = Some(Random.alphanumeric.take(10).mkString("")) + ) + ) + ) + val updatedNativeConfig = updatedBuild.options.scalaNativeOptions.config + + val changedAfterConfigUpdate = + NativeBuilderHelper.shouldBuildIfChanged( + updatedBuild, + updatedNativeConfig, + destPath, + nativeWorkDir + ) + expect(changedAfterConfigUpdate) + } + } + +} diff --git a/modules/cli/src/main/scala/scala/cli/commands/Package.scala b/modules/cli/src/main/scala/scala/cli/commands/Package.scala index 34f70616ef..6a4c0f05e8 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/Package.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/Package.scala @@ -1,13 +1,7 @@ package scala.cli.commands import caseapp._ -import coursier.launcher.{ - AssemblyGenerator, - BootstrapGenerator, - ClassPathEntry, - Parameters, - Preamble -} +import coursier.launcher._ import org.scalajs.linker.interface.StandardConfig import packager.config._ import packager.deb.DebianPackage @@ -26,7 +20,7 @@ import java.util.zip.{ZipEntry, ZipOutputStream} import scala.build.EitherCps.{either, value} import scala.build.errors.BuildException -import scala.build.internal.ScalaJsConfig +import scala.build.internal.{NativeBuilderHelper, ScalaJsConfig} import scala.build.options.{PackageType, Platform} import scala.build.{Build, Inputs, Logger, Os} import scala.cli.commands.OptionsHelper._ @@ -549,18 +543,22 @@ object Package extends ScalaCommand[PackageOptions] { ): Unit = { os.makeDir.all(nativeWorkDir) + val changed = NativeBuilderHelper.shouldBuildIfChanged(build, nativeConfig, dest, nativeWorkDir) + + if (changed) + withLibraryJar(build, dest.last.stripSuffix(".jar")) { mainJar => + val config = sn.Config.empty + .withCompilerConfig(nativeConfig) + .withMainClass(mainClass + "$") + .withClassPath(mainJar +: build.artifacts.classPath) + .withWorkdir(nativeWorkDir.toNIO) + .withLogger(nativeLogger) + + Scope { implicit scope => + sn.Build.build(config, dest.toNIO) + } - withLibraryJar(build, dest.last.stripSuffix(".jar")) { mainJar => - val config = sn.Config.empty - .withCompilerConfig(nativeConfig) - .withMainClass(mainClass + "$") - .withClassPath(mainJar +: build.artifacts.classPath) - .withWorkdir(nativeWorkDir.toNIO) - .withLogger(nativeLogger) - - Scope { implicit scope => - sn.Build.build(config, dest.toNIO) + NativeBuilderHelper.updateOutputSha(dest, nativeWorkDir) } - } } } diff --git a/modules/cli/src/main/scala/scala/cli/commands/Run.scala b/modules/cli/src/main/scala/scala/cli/commands/Run.scala index 733b906e7c..424d2930bf 100644 --- a/modules/cli/src/main/scala/scala/cli/commands/Run.scala +++ b/modules/cli/src/main/scala/scala/cli/commands/Run.scala @@ -202,11 +202,8 @@ object Run extends ScalaCommand[RunOptions] { workDir: os.Path, logger: sn.Logger )(f: os.Path => T): T = { - val dest = os.temp(prefix = "main", suffix = if (Properties.isWin) ".exe" else "") - try { - Package.buildNative(build, mainClass, dest, config, workDir, logger) - f(dest) - } - finally if (os.exists(dest)) os.remove(dest) + val dest = workDir / s"main${if (Properties.isWin) ".exe" else ""}" + Package.buildNative(build, mainClass, dest, config, workDir, logger) + f(dest) } }