diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d446800b..ab0a1bc07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,8 @@ We apologize for the inconvenience. * `smithy4sUpdateLSPConfig`: Replace `imports` with `sources` to be more in line with idiomatic smithy-build config in https://github.com/disneystreaming/smithy4s/pull/1518 (see https://github.com/disneystreaming/smithy4s/issues/1459) * Update smithy: 1.45.0 to 1.49.0 (binary breaking) in https://github.com/disneystreaming/smithy4s/pull/1485 * Rendered type aliases are now sorted alphabetically in https://github.com/disneystreaming/smithy4s/pull/1523 -* Adds handlers construct to facilitate the decoupling of operation implementations in https://github.com/disneystreaming/smithy4s/pull/1522 +* Add handlers construct to facilitate the decoupling of operation implementations in https://github.com/disneystreaming/smithy4s/pull/1522 +* Improve cache in code generation (sbt) in https://github.com/disneystreaming/smithy4s/pull/1499 # 0.18.18 diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-newtype-flatten/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-newtype-flatten/test index d853c9a95..c813c07d5 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-newtype-flatten/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-newtype-flatten/test @@ -2,7 +2,7 @@ > run # This new type was not flattened because it has range constraints -$ exists smithy_output/com/amazonaws/dynamodb/ListTablesInputLimit.scala +$ exists smithy_output/smithy4s/com/amazonaws/dynamodb/ListTablesInputLimit.scala # Flattened and removed by the projection transformer --$ exists smithy_output/com/amazonaws/dynamodb/Long.scala +-$ exists smithy_output/smithy4s/com/amazonaws/dynamodb/Long.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-specs/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-specs/test index 102f0d70c..e58e612c4 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-specs/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/aws-specs/test @@ -1,3 +1,3 @@ # check if smithy4sCodegen works > smithy4sCodegen -$ exists target/scala-2.13/src_managed/main/scala/com/amazonaws/dynamodb/AttributeValue.scala +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/com/amazonaws/dynamodb/AttributeValue.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/custom-settings/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/custom-settings/test index 4372b5970..0e740f87c 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/custom-settings/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/custom-settings/test @@ -1,10 +1,10 @@ # check if smithy4sCodegen works > show p1/smithy4sAllExternalDependencies > p1/compile -$ exists p1/smithy_output/aws/iam/ActionPermissionDescription.scala -$ exists p1/smithy_output/smithy4s/example/ObjectService.scala +$ exists p1/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala +$ exists p1/smithy_output/smithy4s/smithy4s/example/ObjectService.scala > p2/compile -$ exists p2/smithy_output/aws/iam/ActionPermissionDescription.scala -$ exists p2/smithy_output/smithy4s/example/ObjectService.scala --$ exists p2/smithy_output/smithy4s/toexclude/StructureToExclude.scala +$ exists p2/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala +$ exists p2/smithy_output/smithy4s/smithy4s/example/ObjectService.scala +-$ exists p2/smithy_output/smithy4s/smithy4s/toexclude/StructureToExclude.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/defaults/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/defaults/test index 91e32b89f..03ff1b43e 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/defaults/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/defaults/test @@ -1,6 +1,6 @@ # check if smithy4sCodegen works > compile -$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ObjectService.scala +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/ObjectService.scala $ exists target/scala-2.13/resource_managed/main/smithy4s.example.ObjectService.json # check if code can run, this can reveal runtime issues @@ -8,11 +8,11 @@ $ exists target/scala-2.13/resource_managed/main/smithy4s.example.ObjectService. > run $ copy-file example-added.smithy src/main/smithy/example-added.smithy > compile -$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/Added.scala +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/Added.scala # ensuring that removing existing files removes their outputs $ delete src/main/smithy/example.smithy -> compile > smithy4sUpdateLSPConfig -> checkSmithyBuild \ No newline at end of file +> checkSmithyBuild diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/test index 7e6822104..1b03c2fca 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/dependencies-only/test @@ -1,3 +1,3 @@ # check if smithy4sCodegen works > p1/compile -$ exists p1/smithy_output/aws/iam/ActionPermissionDescription.scala +$ exists p1/smithy_output/smithy4s/aws/iam/ActionPermissionDescription.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/extra-configs/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/extra-configs/test index b1811de51..4815ec114 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/extra-configs/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/extra-configs/test @@ -1,7 +1,7 @@ # check if main sources are generated > compile -$ exists target/scala-2.13/src_managed/main/scala/example/ExampleStruct.scala +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ExampleStruct.scala # check if test sources are generated > Test/compile -$ exists target/scala-2.13/src_managed/test/scala/testexample/TestStruct.scala +$ exists target/scala-2.13/src_managed/test/scala/smithy4s/testexample/TestStruct.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/test index 322e41d85..883b4d0ac 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-aws/test @@ -1,5 +1,5 @@ # check if smithy4sCodegen works with libraries that were built with Smithy4s > show bar/smithy4sAllExternalDependencies > compile -$ exists foo/target/scala-2.13/src_managed/main/scala/foo/Lambda.scala -$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Lambda.scala +$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Lambda.scala +$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Lambda.scala diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test index a108f522d..183e549f6 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule-staged/test @@ -2,8 +2,8 @@ > foo/publishLocal > upstream/publishLocal > bar/compile -$ exists bar/target/scala-2.13/src_managed/main/scala/bar/Bar.scala -$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Foo.scala +$ exists bar/target/scala-2.13/src_managed/main/scala/smithy4s/bar/Bar.scala +$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala # check if code can run, this can reveal runtime issues# such as initialization errors > bar/run diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule/test index 264942144..e48a23e0e 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/multimodule/test @@ -1,11 +1,10 @@ # check if smithy4sCodegen works in multimodule contexts > compile -$ exists bar/target/scala-2.13/src_managed/main/scala/bar/Bar.scala -$ absent bar/target/scala-2.13/src_managed/main/scala/foo/Foo.scala -$ absent bar/target/scala-2.13/src_managed/main/scala/foodir/FooDir.scala -$ exists foo/target/scala-2.13/src_managed/main/scala/foo/Foo.scala -$ exists foo/target/scala-2.13/src_managed/main/scala/foodir/FooDir.scala - +$ exists bar/target/scala-2.13/src_managed/main/scala/smithy4s/bar/Bar.scala +$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala +$ absent bar/target/scala-2.13/src_managed/main/scala/smithy4s/foodir/FooDir.scala +$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foo/Foo.scala +$ exists foo/target/scala-2.13/src_managed/main/scala/smithy4s/foodir/FooDir.scala # check if code can run, this can reveal runtime issues# such as initialization errors > bar/run @@ -14,4 +13,4 @@ $ copy-file a.scala foo/src/main/scala/a.scala > bar/run > smithy4sUpdateLSPConfig -> checkSmithyBuild \ No newline at end of file +> checkSmithyBuild diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/scala3/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/scala3/test index 3cd2331f1..c6fef7253 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/scala3/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/scala3/test @@ -1,10 +1,10 @@ # check if smithy4sCodegen works > compile -$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/BadRequest.scala -$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/InternalServerError.scala -$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/ErrorService.scala -$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/errors/package.scala +$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/BadRequest.scala +$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/InternalServerError.scala +$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/ErrorService.scala +$ exists target/scala-3.3.0/src_managed/main/scala/smithy4s/smithy4s/errors/package.scala # check if code can run > run diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-build/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-build/test index 0f186a0a1..442fb337b 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-build/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-build/test @@ -1,5 +1,5 @@ # check if smithy4sCodegen works > compile -$ exists target/scala-2.13/src_managed/main/scala/smithy4s/example/ObjectService.scala +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy4s/example/ObjectService.scala $ exists target/scala-2.13/resource_managed/main/smithy4s.example.ObjectService.json -> checkOpenApi \ No newline at end of file +> checkOpenApi diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-rules/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-rules/test index 764d41bbb..b2a89b735 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-rules/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/smithy-rules/test @@ -1,3 +1,3 @@ # check if smithy4sCodegen works > compile -$ exists target/scala-2.13/src_managed/main/scala/smithy/rules +$ exists target/scala-2.13/src_managed/main/scala/smithy4s/smithy/rules diff --git a/modules/codegen-plugin/src/sbt-test/codegen-plugin/wildcard-config/test b/modules/codegen-plugin/src/sbt-test/codegen-plugin/wildcard-config/test index 9dda2172e..42a1ffba5 100644 --- a/modules/codegen-plugin/src/sbt-test/codegen-plugin/wildcard-config/test +++ b/modules/codegen-plugin/src/sbt-test/codegen-plugin/wildcard-config/test @@ -19,10 +19,3 @@ $ exists target/scala-2.13/resource_managed/main/META-INF/smithy/generated-metad > ++3.3.0 root/compile $ exists target/scala-3.3.0/src_managed/main/smithy/generated-metadata.smithy $ exists target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy - -# ensure metadata file is re-generated after deleting -$ delete target/scala-2.13/src_managed/main/smithy/generated-metadata.smithy -$ delete target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy -> ++3.3.0 root/compile -$ exists target/scala-3.3.0/src_managed/main/smithy/generated-metadata.smithy -$ exists target/scala-3.3.0/resource_managed/main/META-INF/smithy/generated-metadata.smithy diff --git a/modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala b/modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala index 7d60c16f3..0cd934a79 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/JsonConverters.scala @@ -30,24 +30,30 @@ private[smithy4s] object JsonConverters { // This serialises a path by providing a hash of the content it points to. // Because the hash is part of the Json, this allows SBT to detect when a file // changes and invalidate its relevant caches, leading to a call to Smithy4s' code generator. - implicit val pathFormat: JsonFormat[os.Path] = - BasicJsonProtocol.projectFormat[os.Path, HashFileInfo]( + implicit val pathRefFormat: JsonFormat[PathRef] = + BasicJsonProtocol.projectFormat[PathRef, HashFileInfo]( p => { - if (os.isFile(p)) FileInfo.hash(p.toIO) + if (os.isFile(p.underlying)) FileInfo.hash(p.underlying.toIO) else // If the path is a directory, we get the hashes of all files // then hash the concatenation of the hash's bytes. FileInfo.hash( - p.toIO, + p.underlying.toIO, Hash( - os.walk(p) + os.walk(p.underlying) .map(_.toIO) .map(Hash(_)) .foldLeft(Array.emptyByteArray)(_ ++ _) ) ) }, - hash => os.Path(hash.file) + hash => PathRef(os.Path(hash.file)) + ) + + implicit val pathFormat: JsonFormat[os.Path] = + BasicJsonProtocol.projectFormat[os.Path, String]( + p => p.toString, + str => os.Path(str) ) implicit val fileTypeFormat: JsonFormat[FileType] = @@ -61,11 +67,15 @@ private[smithy4s] object JsonConverters { ) // format: off - type GenTarget = List[os.Path] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[os.Path] :*: Option[os.Path] :*: LNil + type GenTarget = List[PathRef] :*: os.Path :*: os.Path :*: Set[FileType] :*: Boolean:*: Option[Set[String]] :*: Option[Set[String]] :*: List[String] :*: List[String] :*: List[String] :*: List[PathRef] :*: Option[PathRef] :*: LNil // format: on + + // `output` and `resourceOutput` are intentionally serialized as paths + // instead of PathRefs. This is to avoid hashing the directories that are generated by smithy4s anyway + // See https://github.com/disneystreaming/smithy4s/issues/1495 for reference on this decision implicit val codegenArgsIso = LList.iso[CodegenArgs, GenTarget]( { ca: CodegenArgs => - ("specs", ca.specs) :*: + ("specs", ca.specs.map(PathRef(_))) :*: ("output", ca.output) :*: ("resourceOutput", ca.resourceOutput) :*: ("skip", ca.skip) :*: @@ -75,8 +85,8 @@ private[smithy4s] object JsonConverters { ("repositories", ca.repositories) :*: ("dependencies", ca.dependencies) :*: ("transformers", ca.transformers) :*: - ("localJars", ca.localJars) :*: - ("smithyBuild", ca.smithyBuild) :*: + ("localJars", ca.localJars.map(PathRef(_))) :*: + ("smithyBuild", ca.smithyBuild.map(PathRef(_))) :*: LNil }, { @@ -93,7 +103,7 @@ private[smithy4s] object JsonConverters { (_, localJars) :*: (_, smithyBuild) :*: LNil => CodegenArgs( - specs, + specs.map(_.underlying), output, resourceOutput, skip, @@ -103,8 +113,8 @@ private[smithy4s] object JsonConverters { repositories, dependencies, transformers, - localJars, - smithyBuild + localJars.map(_.underlying), + smithyBuild.map(_.underlying) ) } ) diff --git a/modules/codegen-plugin/src/smithy4s/codegen/PathRef.scala b/modules/codegen-plugin/src/smithy4s/codegen/PathRef.scala new file mode 100644 index 000000000..f4bd5ee9d --- /dev/null +++ b/modules/codegen-plugin/src/smithy4s/codegen/PathRef.scala @@ -0,0 +1,19 @@ +/* + * Copyright 2021-2024 Disney Streaming + * + * Licensed under the Tomorrow Open Source Technology License, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://disneystreaming.github.io/TOST-1.0.txt + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package smithy4s.codegen + +final case class PathRef(underlying: os.Path) diff --git a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala index 5bbb81d22..623fc3e9b 100644 --- a/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala +++ b/modules/codegen-plugin/src/smithy4s/codegen/Smithy4sCodegenPlugin.scala @@ -408,14 +408,15 @@ object Smithy4sCodegenPlugin extends AutoPlugin { (inputDirs ++ generatedFiles) .filter(_.exists()) .toList - val outputPath = (conf / smithy4sOutputDir).value + val outputPath = (conf / smithy4sOutputDir).value / "smithy4s" val resourceOutputPath = (conf / smithy4sResourceDir).value val allowedNamespaces = (conf / smithy4sAllowedNamespaces).?.value.map(_.toSet) val excludedNamespaces = (conf / smithy4sExcludedNamespaces).?.value.map(_.toSet) val localJars = - (conf / smithy4sAllDependenciesAsJars).value.map(os.Path(_)).toList + (conf / smithy4sAllDependenciesAsJars).value.toList.sorted + .map(p => os.Path(p)) val res = (conf / resolvers).value.toList.collect { case m: MavenRepository => m.root @@ -425,12 +426,17 @@ object Smithy4sCodegenPlugin extends AutoPlugin { val skipResources: Set[FileType] = if ((conf / smithy4sSmithyLibrary).value) Set.empty else Set(FileType.Resource) + val skipSet = skipResources val filePaths = inputFiles.map(_.getAbsolutePath()) + + val specs = filePaths.sorted.map(p => os.Path(p)).toList + val smithyBuildValue = (conf / smithyBuild).value.map(os.Path(_)) + val codegenArgs = CodegenArgs( - filePaths.map(os.Path(_)).toList, + specs, output = os.Path(outputPath), resourceOutput = os.Path(resourceOutputPath), skip = skipSet, @@ -453,11 +459,13 @@ object Smithy4sCodegenPlugin extends AutoPlugin { s.cacheStoreFactory.make("output") ) { case ((inputChanged, args), outputs) => if (inputChanged || outputs.isEmpty) { + s.log.debug("Regenerating managed sources") val resPaths = smithy4s.codegen.Codegen .generateToDisk(args) .toList resPaths.map(path => new File(path.toString)) } else { + s.log.debug("Using cached version of outputs") outputs.getOrElse(Seq.empty) } } diff --git a/modules/codegen/src/smithy4s/codegen/CodegenArgs.scala b/modules/codegen/src/smithy4s/codegen/CodegenArgs.scala index b59d2f20f..b344ceae0 100644 --- a/modules/codegen/src/smithy4s/codegen/CodegenArgs.scala +++ b/modules/codegen/src/smithy4s/codegen/CodegenArgs.scala @@ -37,6 +37,7 @@ final case class CodegenArgs( def skipOpenapi: Boolean = skip(FileType.Openapi) def skipResources: Boolean = skip(FileType.Resource) def skipProto: Boolean = skip(FileType.Proto) + } sealed abstract class FileType(val name: String) diff --git a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala index bd8727e08..67e030ef7 100644 --- a/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala +++ b/modules/mill-codegen-plugin/src/smithy4s/codegen/mill/Smithy4sModule.scala @@ -192,6 +192,7 @@ trait Smithy4sModule extends ScalaModule { val specFiles = (smithy4sGeneratedSmithyFiles() ++ smithy4sInputDirs()) .map(_.path) .filter(os.exists(_)) + .toList val scalaOutput = smithy4sOutputDir().path val resourcesOutput = smithy4sResourceOutputDir().path @@ -209,10 +210,13 @@ trait Smithy4sModule extends ScalaModule { val smithyBuildFile = smithyBuild().map(_.path) val allLocalJars = - smithy4sAllDependenciesAsJars().map(_.path).iterator.to(List) + smithy4sAllDependenciesAsJars() + .map(_.path) + .iterator + .to(List) val args = CodegenArgs( - specs = specFiles.toList, + specs = specFiles, output = scalaOutput, resourceOutput = resourcesOutput, skip = skipSet,