From 36a061d7a936c55ee6a507259f6cee70320f2235 Mon Sep 17 00:00:00 2001 From: Ryan Peters Date: Fri, 15 Jan 2021 16:10:17 -0600 Subject: [PATCH 1/5] Update scrimage to 4.x from 2.x --- build.sbt | 2 +- project/dependencies.sbt | 8 ++++---- src/main/scala/microsites/util/MicrositeHelper.scala | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/build.sbt b/build.sbt index f98984fa..6e1894d0 100644 --- a/build.sbt +++ b/build.sbt @@ -43,7 +43,7 @@ lazy val pluginSettings: Seq[Def.Setting[_]] = Seq( "com.47deg" %% "github4s" % "0.24.0", "net.jcazevedo" %% "moultingyaml" % "0.4.2", "com.lihaoyi" %% "scalatags" % "0.9.2", - "com.sksamuel.scrimage" %% "scrimage-core" % "2.1.8", + "com.sksamuel.scrimage" %% "scrimage-scala" % "4.0.12", "org.scalatestplus" %% "scalacheck-1-14" % "3.1.4.0" % Test ), scriptedLaunchOpts ++= Seq( diff --git a/project/dependencies.sbt b/project/dependencies.sbt index 095edc4d..cd13a78e 100644 --- a/project/dependencies.sbt +++ b/project/dependencies.sbt @@ -5,8 +5,8 @@ unmanagedResourceDirectories in Compile += baseDirectory.in(ThisBuild).value.getParentFile / "src" / "main" / "resources" libraryDependencies ++= Seq( - "com.47deg" %% "github4s" % "0.24.0", - "net.jcazevedo" %% "moultingyaml" % "0.4.2", - "com.lihaoyi" %% "scalatags" % "0.9.2", - "com.sksamuel.scrimage" %% "scrimage-core" % "2.1.8" + "com.47deg" %% "github4s" % "0.24.0", + "net.jcazevedo" %% "moultingyaml" % "0.4.2", + "com.lihaoyi" %% "scalatags" % "0.9.2", + "com.sksamuel.scrimage" %% "scrimage-scala" % "4.0.12" ) diff --git a/src/main/scala/microsites/util/MicrositeHelper.scala b/src/main/scala/microsites/util/MicrositeHelper.scala index a8ccf42e..a6df271b 100644 --- a/src/main/scala/microsites/util/MicrositeHelper.scala +++ b/src/main/scala/microsites/util/MicrositeHelper.scala @@ -20,7 +20,8 @@ import java.io.File import java.net.URL import cats.syntax.either._ -import com.sksamuel.scrimage._ +import com.sksamuel.scrimage.ImmutableImage +import com.sksamuel.scrimage.implicits._ import microsites.util.YamlFormats._ import microsites._ import microsites.layouts._ @@ -234,7 +235,7 @@ class MicrositeHelper(config: MicrositeSettings) { (new File(s"$targetDir$jekyllDir/img/$name"), size) } .map { case (file, (width, height)) => - Image.fromFile(sourceFile.toFile).scaleTo(width, height).output(file) + ImmutableImage.loader.fromFile(sourceFile.toFile).scaleTo(width, height).output(file) } } From 4c2567d9526ad9597cb3a72e0f9fb600c48fa75c Mon Sep 17 00:00:00 2001 From: Ryan Peters Date: Tue, 19 Jan 2021 16:07:11 -0600 Subject: [PATCH 2/5] Rewrite copyJARResourcesTo to work with arbitrary filesystems --- .../scala/microsites/ioops/FileWriter.scala | 66 ++++++++----------- .../microsites/util/MicrositeHelper.scala | 5 +- 2 files changed, 29 insertions(+), 42 deletions(-) diff --git a/src/main/scala/microsites/ioops/FileWriter.scala b/src/main/scala/microsites/ioops/FileWriter.scala index 0d2bc26e..d191bc8d 100644 --- a/src/main/scala/microsites/ioops/FileWriter.scala +++ b/src/main/scala/microsites/ioops/FileWriter.scala @@ -16,17 +16,14 @@ package microsites.ioops -import java.io.{File, FileOutputStream} +import java.io.File import java.net.URL +import java.nio.file.{FileSystems, Files, Path, Paths} import java.nio.file.Files.{copy => fcopy} -import java.nio.file.Path import java.nio.file.StandardCopyOption.REPLACE_EXISTING -import java.util.zip.ZipInputStream +import java.{util => ju} -import cats.instances.either._ -import cats.instances.list._ -import cats.syntax.either._ -import cats.syntax.traverse._ +import cats.implicits._ import microsites.Exceptions._ import syntax._ @@ -86,51 +83,40 @@ class FileWriter { def copyMultipleFiles(files: List[File]): IOResult[List[Path]] = files.traverse(copySingleFile) + println(sourcePath) for { files <- FileReader.fetchFilesRecursivelyFromPath(sourcePath) + _ = println(s"files: $files") paths <- copyMultipleFiles(files) + _ = println(s"paths: $paths") } yield paths } - def copyJARResourcesTo( - jarUrl: URL, - output: String, + /** Copies files from an arbitrary FileSystem (dir, JAR, ZIP) defined at the listed URL */ + def copyResourcesTo( + url: URL, + outputPath: String, filter: String = "" - ): Either[IOException, Unit] = + ) = { + val emptyMap = new ju.HashMap[String, Any] Either - .catchNonFatal { - output.fixPath.toFile.mkdir() - - val zipIn = new ZipInputStream(jarUrl.openStream()) - - val buffer = new Array[Byte](1024) - Stream.continually(zipIn.getNextEntry).takeWhile(_ != null).foreach { entry => - val newFile = IOUtils.file(output + File.separator + entry.getName) - - ( - entry.isDirectory, - !newFile.exists() && - newFile.getAbsolutePath.startsWith(s"$output$filter") - ) match { - case (true, true) => createDir(newFile) - case (true, _) => - case (false, true) => - createFile(newFile) - - val fos = new FileOutputStream(newFile) - - Stream.continually(zipIn.read(buffer)).takeWhile(_ != -1).foreach { count => - fos.write(buffer, 0, count) - } - - fos.close() - case _ => + .catchNonFatal(FileSystems.newFileSystem(url.toURI, emptyMap)) + .flatMap { fs => + Either + .catchNonFatal { + val filesInPath = fs.getPath(filter).iterator + + filesInPath.forEachRemaining { path => + Files.copy(path, Paths.get(outputPath), REPLACE_EXISTING) + } } - } + .leftMap { e => fs.close(); e } + .map(_ => fs.close()) } .leftMap(e => - IOException(s"Error copying resources from $jarUrl to directory $output", Some(e)) + IOException(s"Error copying resources from $url to directory $outputPath", Some(e)) ) + } } object FileWriter extends FileWriter diff --git a/src/main/scala/microsites/util/MicrositeHelper.scala b/src/main/scala/microsites/util/MicrositeHelper.scala index a6df271b..e1267f7c 100644 --- a/src/main/scala/microsites/util/MicrositeHelper.scala +++ b/src/main/scala/microsites/util/MicrositeHelper.scala @@ -61,8 +61,9 @@ class MicrositeHelper(config: MicrositeSettings) { output: String, filter: String = "" ): Either[Exceptions.IOException, Any] = - copyJARResourcesTo(jarUrl, output, filter) - .orElse(copyFilesRecursively(jarUrl.getFile + filter, output + filter)) + copyResourcesTo(jarUrl, output, filter).orElse( + copyFilesRecursively(jarUrl.getFile + filter, output + filter) + ) def createResources(resourceManagedDir: File): List[File] = { From 24d93b929e42fcc3dad7d1e23e58f080cfc3ce35 Mon Sep 17 00:00:00 2001 From: Ryan Peters Date: Tue, 19 Jan 2021 16:07:41 -0600 Subject: [PATCH 3/5] Add workaround for scrimage 4 classloader issue --- .../microsites/util/MicrositeHelper.scala | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/main/scala/microsites/util/MicrositeHelper.scala b/src/main/scala/microsites/util/MicrositeHelper.scala index e1267f7c..0d0dcef5 100644 --- a/src/main/scala/microsites/util/MicrositeHelper.scala +++ b/src/main/scala/microsites/util/MicrositeHelper.scala @@ -20,6 +20,7 @@ import java.io.File import java.net.URL import cats.syntax.either._ +import com.sksamuel.scrimage.nio.ImageReader import com.sksamuel.scrimage.ImmutableImage import com.sksamuel.scrimage.implicits._ import microsites.util.YamlFormats._ @@ -231,13 +232,26 @@ class MicrositeHelper(config: MicrositeSettings) { createFile(sourceFile) - (faviconFilenames zip faviconSizes) - .map { case (name, size) => - (new File(s"$targetDir$jekyllDir/img/$name"), size) - } - .map { case (file, (width, height)) => - ImmutableImage.loader.fromFile(sourceFile.toFile).scaleTo(width, height).output(file) - } + //This is a dirty classloader hack to allow the latest version of Scrimage to work. + //This plugin's default classloader is limited and cannot load ImageReader instances. + //We get a different ClassLoader that will work, replace it, and then set it back after we're done. + //Will be fixed in Scrimage 4.1.0, see: https://github.com/sksamuel/scrimage/issues/217 + val desiredCL: ClassLoader = classOf[ImageReader].getClassLoader + val currentCL = Thread.currentThread.getContextClassLoader() + + try { + Thread.currentThread.setContextClassLoader(desiredCL) + + (faviconFilenames zip faviconSizes) + .map { case (name, size) => + (new File(s"$targetDir$jekyllDir/img/$name"), size) + } + .map { case (file, (width, height)) => + ImmutableImage.loader.fromFile(sourceFile.toFile).scaleTo(width, height).output(file) + } + } finally + //Reset the classloader to what it was previously + Thread.currentThread.setContextClassLoader(currentCL) } def copyConfigurationFile(sourceDir: File, targetDir: File): Unit = { From 69446731576598c038805c93863f0eefe839b256 Mon Sep 17 00:00:00 2001 From: Ryan Peters Date: Tue, 19 Jan 2021 16:17:36 -0600 Subject: [PATCH 4/5] Remove some old printlns --- src/main/scala/microsites/ioops/FileWriter.scala | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/scala/microsites/ioops/FileWriter.scala b/src/main/scala/microsites/ioops/FileWriter.scala index d191bc8d..250a9fb6 100644 --- a/src/main/scala/microsites/ioops/FileWriter.scala +++ b/src/main/scala/microsites/ioops/FileWriter.scala @@ -83,12 +83,9 @@ class FileWriter { def copyMultipleFiles(files: List[File]): IOResult[List[Path]] = files.traverse(copySingleFile) - println(sourcePath) for { files <- FileReader.fetchFilesRecursivelyFromPath(sourcePath) - _ = println(s"files: $files") paths <- copyMultipleFiles(files) - _ = println(s"paths: $paths") } yield paths } From 17be889e14304915cdaf61babbad0614b694e1d0 Mon Sep 17 00:00:00 2001 From: Ryan Peters Date: Wed, 20 Jan 2021 14:01:10 -0600 Subject: [PATCH 5/5] Restructure FileSystem resource copying --- .../scala/microsites/ioops/FileWriter.scala | 40 ++++++----- .../microsites/util/MicrositeHelper.scala | 66 ++++++++++++------- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/src/main/scala/microsites/ioops/FileWriter.scala b/src/main/scala/microsites/ioops/FileWriter.scala index 250a9fb6..d32fae71 100644 --- a/src/main/scala/microsites/ioops/FileWriter.scala +++ b/src/main/scala/microsites/ioops/FileWriter.scala @@ -18,8 +18,7 @@ package microsites.ioops import java.io.File import java.net.URL -import java.nio.file.{FileSystems, Files, Path, Paths} -import java.nio.file.Files.{copy => fcopy} +import java.nio.file.{FileSystem, Files, Path} import java.nio.file.StandardCopyOption.REPLACE_EXISTING import java.{util => ju} @@ -64,7 +63,7 @@ class FileWriter { def safeCopy: Either[Throwable, Path] = Either - .catchNonFatal(fcopy(input.toPath, output.toPath, REPLACE_EXISTING)) + .catchNonFatal(Files.copy(input.toPath, output.toPath, REPLACE_EXISTING)) (for { result <- createFile(output) @@ -90,29 +89,28 @@ class FileWriter { } /** Copies files from an arbitrary FileSystem (dir, JAR, ZIP) defined at the listed URL */ - def copyResourcesTo( - url: URL, - outputPath: String, - filter: String = "" + def copyResourcesFromFileSystem( + fs: FileSystem, + outputPath: Path, + filters: List[String] ) = { - val emptyMap = new ju.HashMap[String, Any] Either - .catchNonFatal(FileSystems.newFileSystem(url.toURI, emptyMap)) - .flatMap { fs => - Either - .catchNonFatal { - val filesInPath = fs.getPath(filter).iterator - - filesInPath.forEachRemaining { path => - Files.copy(path, Paths.get(outputPath), REPLACE_EXISTING) + .catchNonFatal { + filters.foreach { filter => + val fsPath = fs.getPath(filter) + if (Files.exists(fsPath)) { + Files.walk(fsPath).forEachOrdered { path => + val dest = outputPath.resolve(path.toString.stripPrefix("/")) + try Option(dest.getParent()).foreach(Files.createDirectories(_)) + finally Files.copy(path, dest, REPLACE_EXISTING) } } - .leftMap { e => fs.close(); e } - .map(_ => fs.close()) + } + } + .leftMap { e => + e.printStackTrace() + IOException(s"Error copying resources from JAR to directory $outputPath", Some(e)) } - .leftMap(e => - IOException(s"Error copying resources from $url to directory $outputPath", Some(e)) - ) } } diff --git a/src/main/scala/microsites/util/MicrositeHelper.scala b/src/main/scala/microsites/util/MicrositeHelper.scala index 0d0dcef5..15f6a4af 100644 --- a/src/main/scala/microsites/util/MicrositeHelper.scala +++ b/src/main/scala/microsites/util/MicrositeHelper.scala @@ -17,9 +17,11 @@ package microsites.util import java.io.File -import java.net.URL +import java.net.{URI, URL} +import java.nio.file.{FileSystem, FileSystems, Paths} +import java.util.HashMap -import cats.syntax.either._ +import cats.implicits._ import com.sksamuel.scrimage.nio.ImageReader import com.sksamuel.scrimage.ImmutableImage import com.sksamuel.scrimage.implicits._ @@ -57,32 +59,52 @@ class MicrositeHelper(config: MicrositeSettings) { MicrositeFavicon(filename, s"${width}x$height") } - def copyJAROrFolder( - jarUrl: URL, - output: String, - filter: String = "" - ): Either[Exceptions.IOException, Any] = - copyResourcesTo(jarUrl, output, filter).orElse( - copyFilesRecursively(jarUrl.getFile + filter, output + filter) - ) + def copyResources( + resourcesJarFSOrPath: Either[FileSystem, java.nio.file.Path], + outputPath: java.nio.file.Path, + filters: List[String] + ): Either[Exceptions.IOException, Any] = resourcesJarFSOrPath match { + case Left(fs) => + println(s"Copying resources from sbt-microsite JAR") + val result = copyResourcesFromFileSystem(fs, outputPath, filters) + fs.close() + result + case Right(path) => + println(s"Copying resources from local path $path") + filters.traverse { filter => + copyFilesRecursively(path.toString, outputPath.toString) + } + } def createResources(resourceManagedDir: File): List[File] = { val targetDir: String = resourceManagedDir.getAbsolutePath.ensureFinalSlash val pluginURL: URL = getClass.getProtectionDomain.getCodeSource.getLocation - - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "_sass") - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "css") - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "img") - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "js") - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "highlight/highlight.pack.js") - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "highlight/LICENSE") - copyJAROrFolder( - pluginURL, - s"$targetDir$jekyllDir/", - s"highlight/styles/${config.visualSettings.highlightTheme}.css" + val pluginPath = Paths.get(pluginURL.getPath) + val outputPath = Paths.get(s"$targetDir$jekyllDir/") + + val filters = List( + "_sass", + "css", + "img", + "js", + "highlight/highlight.pack.js", + "highlight/LICENSE", + s"highlight/styles/${config.visualSettings.highlightTheme}.css", + "plugins" ) - copyJAROrFolder(pluginURL, s"$targetDir$jekyllDir/", "plugins") + + //If resources are in a JAR, we want to expose that as a FileSystem + //Otherwise we will just use the raw path + val pathOrFS: Either[FileSystem, java.nio.file.Path] = + if (pluginPath.getFileName.toString.endsWith(".jar")) { + val uri = URI.create("jar:file:" + pluginPath.toString) + FileSystems.newFileSystem(uri, new HashMap[String, Any]()).asLeft + } else { + pluginPath.asRight + } + + copyResources(pathOrFS, outputPath, filters) copyFilesRecursively( config.fileLocations.micrositeImgDirectory.getAbsolutePath,