Improve SBT caching during code generation (#1499)

CHANGELOG.md

@@ -20,7 +20,8 @@ We apologize for the inconvenience.
 * `smithy4sUpdateLSPConfig`: Replace `imports` with `sources` to be more in line with idiomatic smithy-build config in https://github.com/disneystreaming/smithy4s/pull/1518 (see https://github.com/disneystreaming/smithy4s/issues/1459)
 * Update smithy: 1.45.0 to 1.49.0 (binary breaking) in https://github.com/disneystreaming/smithy4s/pull/1485
 * Rendered type aliases are now sorted alphabetically in https://github.com/disneystreaming/smithy4s/pull/1523
-* Adds handlers construct to facilitate the decoupling of operation implementations in https://github.com/disneystreaming/smithy4s/pull/1522
+* Add handlers construct to facilitate the decoupling of operation implementations in https://github.com/disneystreaming/smithy4s/pull/1522
+* Improve cache in code generation (sbt) in https://github.com/disneystreaming/smithy4s/pull/1499
 
 # 0.18.18
 

modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-newtype-flatten/test

@@ -2,7 +2,7 @@
 > run
 
 # This new type was not flattened because it has range constraints
-$ exists  smithy_output/com/amazonaws/dynamodb/ListTablesInputLimit.scala
+$ exists  smithy_output/smithy4s/com/amazonaws/dynamodb/ListTablesInputLimit.scala
 
 # Flattened and removed by the projection transformer
--$ exists smithy_output/com/amazonaws/dynamodb/Long.scala
+-$ exists smithy_output/smithy4s/com/amazonaws/dynamodb/Long.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-specs/test

@@ -1,3 +1,3 @@
 # check if smithy4sCodegen works
 > smithy4sCodegen
-$ exists target/scala-2.13/src_managed/main/scala/com/amazonaws/dynamodb/AttributeValue.scala
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/com/amazonaws/dynamodb/AttributeValue.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/custom-settings/test

@@ -1,10 +1,10 @@
 # check if smithy4sCodegen works
 > show p1/smithy4sAllExternalDependencies
 > p1/compile
-$ exists p1/smithy_output/aws/iam/ActionPermissionDescription.scala
-$ exists p1/smithy_output/smithy4s/example/ObjectService.scala
+$ exists p1/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala
+$ exists p1/smithy_output/smithy4s/smithy4s/example/ObjectService.scala
 
 > p2/compile
-$ exists p2/smithy_output/aws/iam/ActionPermissionDescription.scala
-$ exists p2/smithy_output/smithy4s/example/ObjectService.scala
--$ exists p2/smithy_output/smithy4s/toexclude/StructureToExclude.scala
+$ exists p2/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala
+$ exists p2/smithy_output/smithy4s/smithy4s/example/ObjectService.scala
+-$ exists p2/smithy_output/smithy4s/smithy4s/toexclude/StructureToExclude.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/defaults/test

@@ -1,18 +1,18 @@
 # check if smithy4sCodegen works
 > compile
-$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ObjectService.scala
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/ObjectService.scala
 $ exists target/scala-2.13/resource_managed/main/smithy4s.example.ObjectService.json
 
 # check if code can run, this can reveal runtime issues
 # such as initialization errors
 > run
 $ copy-file example-added.smithy src/main/smithy/example-added.smithy
 > compile
-$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/Added.scala
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/Added.scala
 
 # ensuring that removing existing files removes their outputs
 $ delete src/main/smithy/example.smithy
 -> compile
 
 > smithy4sUpdateLSPConfig
-> checkSmithyBuild
+> checkSmithyBuild

modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/test

@@ -1,3 +1,3 @@
 # check if smithy4sCodegen works
 > p1/compile
-$ exists p1/smithy_output/aws/iam/ActionPermissionDescription.scala
+$ exists p1/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/extra-configs/test

@@ -1,7 +1,7 @@
 # check if main sources are generated
 > compile
-$ exists target/scala-2.13/src_managed/main/scala/example/ExampleStruct.scala
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ExampleStruct.scala
 
 # check if test sources are generated
 > Test/compile
-$ exists target/scala-2.13/src_managed/test/scala/testexample/TestStruct.scala
+$ exists target/scala-2.13/src_managed/test/scala/smithy4s/testexample/TestStruct.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/test

@@ -1,5 +1,5 @@
 # check if smithy4sCodegen works with libraries that were built with Smithy4s
 > show bar/smithy4sAllExternalDependencies
 > compile
-$ exists foo/target/scala-2.13/src_managed/main/scala/foo/Lambda.scala
-$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Lambda.scala
+$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Lambda.scala
+$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Lambda.scala

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test

@@ -2,8 +2,8 @@
 > foo/publishLocal
 > upstream/publishLocal
 > bar/compile
-$ exists bar/target/scala-2.13/src_managed/main/scala/bar/Bar.scala
-$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Foo.scala
+$ exists bar/target/scala-2.13/src_managed/main/scala/smithy4s/bar/Bar.scala
+$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala
 
 # check if code can run, this can reveal runtime issues# such as initialization errors
 > bar/run

modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule/test

@@ -1,11 +1,10 @@
 # check if smithy4sCodegen works in multimodule contexts
 > compile
-$ exists bar/target/scala-2.13/src_managed/main/scala/bar/Bar.scala
-$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Foo.scala
-$ absent bar/target/scala-2.13/src_managed/main/scala/foodir/FooDir.scala
-$ exists foo/target/scala-2.13/src_managed/main/scala/foo/Foo.scala
-$ exists foo/target/scala-2.13/src_managed/main/scala/foodir/FooDir.scala
-
+$ exists bar/target/scala-2.13/src_managed/main/scala/smithy4s/bar/Bar.scala
+$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala
+$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foodir/FooDir.scala
+$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala
+$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foodir/FooDir.scala
 # check if code can run, this can reveal runtime issues# such as initialization errors
 > bar/run
 
@@ -14,4 +13,4 @@ $ copy-file a.scala foo/src/main/scala/a.scala
 > bar/run
 
 > smithy4sUpdateLSPConfig
-> checkSmithyBuild
+> checkSmithyBuild

modules/codegen-plugin/src/sbt-test/codegen-plugin/scala3/test

@@ -1,10 +1,10 @@
 # check if smithy4sCodegen works
 > compile
 
-$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/BadRequest.scala
-$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/InternalServerError.scala
-$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/ErrorService.scala
-$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/package.scala
+$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/BadRequest.scala
+$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/InternalServerError.scala
+$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/ErrorService.scala
+$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/package.scala
 
 # check if code can run
 > run

modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-build/test

@@ -1,5 +1,5 @@
 # check if smithy4sCodegen works
 > compile
-$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ObjectService.scala
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/ObjectService.scala
 $ exists target/scala-2.13/resource_managed/main/smithy4s.example.ObjectService.json
-> checkOpenApi
+> checkOpenApi

modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-rules/test

@@ -1,3 +1,3 @@
 # check if smithy4sCodegen works
 > compile
-$ exists target/scala-2.13/src_managed/main/scala/smithy/rules
+$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy/rules

modules/codegen-plugin/src/sbt-test/codegen-plugin/wildcard-config/test

@@ -19,10 +19,3 @@ $ exists target/scala-2.13/resource_managed/main/META-INF/smithy/generated-metad
 > ++3.3.0 root/compile
 $ exists target/scala-3.3.0/src_managed/main/smithy/generated-metadata.smithy
 $ exists target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy
-
-# ensure metadata file is re-generated after deleting
-$ delete target/scala-2.13/src_managed/main/smithy/generated-metadata.smithy
-$ delete target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy
-> ++3.3.0 root/compile
-$ exists target/scala-3.3.0/src_managed/main/smithy/generated-metadata.smithy
-$ exists target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy

modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala

@@ -30,24 +30,30 @@ private[smithy4s] object JsonConverters {
   // This serialises a path by providing a hash of the content it points to.
   // Because the hash is part of the Json, this allows SBT to detect when a file
   // changes and invalidate its relevant caches, leading to a call to Smithy4s' code generator.
-  implicit val pathFormat: JsonFormat[os.Path] =
-    BasicJsonProtocol.projectFormat[os.Path, HashFileInfo](
+  implicit val pathRefFormat: JsonFormat[PathRef] =
+    BasicJsonProtocol.projectFormat[PathRef, HashFileInfo](
       p => {
-        if (os.isFile(p)) FileInfo.hash(p.toIO)
+        if (os.isFile(p.underlying)) FileInfo.hash(p.underlying.toIO)
         else
           // If the path is a directory, we get the hashes of all files
           // then hash the concatenation of the hash's bytes.
           FileInfo.hash(
-            p.toIO,
+            p.underlying.toIO,
             Hash(
-              os.walk(p)
+              os.walk(p.underlying)
                 .map(_.toIO)
                 .map(Hash(_))
                 .foldLeft(Array.emptyByteArray)(_ ++ _)
             )
           )
       },
-      hash => os.Path(hash.file)
+      hash => PathRef(os.Path(hash.file))
+    )
+
+  implicit val pathFormat: JsonFormat[os.Path] =
+    BasicJsonProtocol.projectFormat[os.Path, String](
+      p => p.toString,
+      str => os.Path(str)
     )
 
   implicit val fileTypeFormat: JsonFormat[FileType] =
@@ -61,11 +67,15 @@ private[smithy4s] object JsonConverters {
     )
 
   // format: off
-  type GenTarget = List[os.Path] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[os.Path] :*: Option[os.Path] :*: LNil
+  type GenTarget = List[PathRef] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[PathRef] :*: Option[PathRef] :*: LNil
   // format: on
+
+  // `output` and `resourceOutput` are intentionally serialized as paths
+  // instead of PathRefs. This is to avoid hashing the directories that are generated by smithy4s anyway
+  // See https://github.com/disneystreaming/smithy4s/issues/1495 for reference on this decision
   implicit val codegenArgsIso = LList.iso[CodegenArgs, GenTarget](
     { ca: CodegenArgs =>
-      ("specs", ca.specs) :*:
+      ("specs", ca.specs.map(PathRef(_))) :*:
         ("output", ca.output) :*:
         ("resourceOutput", ca.resourceOutput) :*:
         ("skip", ca.skip) :*:
@@ -75,8 +85,8 @@ private[smithy4s] object JsonConverters {
         ("repositories", ca.repositories) :*:
         ("dependencies", ca.dependencies) :*:
         ("transformers", ca.transformers) :*:
-        ("localJars", ca.localJars) :*:
-        ("smithyBuild", ca.smithyBuild) :*:
+        ("localJars", ca.localJars.map(PathRef(_))) :*:
+        ("smithyBuild", ca.smithyBuild.map(PathRef(_))) :*:
         LNil
     },
     {
@@ -93,7 +103,7 @@ private[smithy4s] object JsonConverters {
           (_, localJars) :*:
           (_, smithyBuild) :*: LNil =>
         CodegenArgs(
-          specs,
+          specs.map(_.underlying),
           output,
           resourceOutput,
           skip,
@@ -103,8 +113,8 @@ private[smithy4s] object JsonConverters {
           repositories,
           dependencies,
           transformers,
-          localJars,
-          smithyBuild
+          localJars.map(_.underlying),
+          smithyBuild.map(_.underlying)
         )
     }
   )

modules/codegen-plugin/src/smithy4s/codegen/PathRef.scala

@@ -0,0 +1,19 @@
+/*
+ *  Copyright 2021-2024 Disney Streaming
+ *
+ *  Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *     https://disneystreaming.github.io/TOST-1.0.txt
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package smithy4s.codegen
+
+final case class PathRef(underlying: os.Path)

modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala

@@ -408,14 +408,15 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
       (inputDirs ++ generatedFiles)
         .filter(_.exists())
         .toList
-    val outputPath = (conf / smithy4sOutputDir).value
+    val outputPath = (conf / smithy4sOutputDir).value / "smithy4s"
     val resourceOutputPath = (conf / smithy4sResourceDir).value
     val allowedNamespaces =
       (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet)
     val excludedNamespaces =
       (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet)
     val localJars =
-      (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList
+      (conf / smithy4sAllDependenciesAsJars).value.toList.sorted
+        .map(p => os.Path(p))
     val res =
       (conf / resolvers).value.toList.collect { case m: MavenRepository =>
         m.root
@@ -425,12 +426,17 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
     val skipResources: Set[FileType] =
       if ((conf / smithy4sSmithyLibrary).value) Set.empty
       else Set(FileType.Resource)
+
     val skipSet = skipResources
 
     val filePaths = inputFiles.map(_.getAbsolutePath())
+
+    val specs = filePaths.sorted.map(p => os.Path(p)).toList
+
     val smithyBuildValue = (conf / smithyBuild).value.map(os.Path(_))
+
     val codegenArgs = CodegenArgs(
-      filePaths.map(os.Path(_)).toList,
+      specs,
       output = os.Path(outputPath),
       resourceOutput = os.Path(resourceOutputPath),
       skip = skipSet,
@@ -453,11 +459,13 @@ object Smithy4sCodegenPlugin extends AutoPlugin {
             s.cacheStoreFactory.make("output")
           ) { case ((inputChanged, args), outputs) =>
             if (inputChanged || outputs.isEmpty) {
+              s.log.debug("Regenerating managed sources")
               val resPaths = smithy4s.codegen.Codegen
                 .generateToDisk(args)
                 .toList
               resPaths.map(path => new File(path.toString))
             } else {
+              s.log.debug("Using cached version of outputs")
               outputs.getOrElse(Seq.empty)
             }
           }

modules/codegen/src/smithy4s/codegen/CodegenArgs.scala

@@ -37,6 +37,7 @@ final case class CodegenArgs(
   def skipOpenapi: Boolean = skip(FileType.Openapi)
   def skipResources: Boolean = skip(FileType.Resource)
   def skipProto: Boolean = skip(FileType.Proto)
+
 }
 
 sealed abstract class FileType(val name: String)

modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala

@@ -192,6 +192,7 @@ trait Smithy4sModule extends ScalaModule {
     val specFiles = (smithy4sGeneratedSmithyFiles() ++ smithy4sInputDirs())
       .map(_.path)
       .filter(os.exists(_))
+      .toList
 
     val scalaOutput = smithy4sOutputDir().path
     val resourcesOutput = smithy4sResourceOutputDir().path
@@ -209,10 +210,13 @@ trait Smithy4sModule extends ScalaModule {
     val smithyBuildFile = smithyBuild().map(_.path)
 
     val allLocalJars =
-      smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List)
+      smithy4sAllDependenciesAsJars()
+        .map(_.path)
+        .iterator
+        .to(List)
 
     val args = CodegenArgs(
-      specs = specFiles.toList,
+      specs = specFiles,
       output = scalaOutput,
       resourceOutput = resourcesOutput,
       skip = skipSet,