Paradox site (#879)

* Paradox magnolify site * Enable site in CI * Fix mapping table * Remove extra setup step
spotify · Jan 9, 2024 · 11561c4 · 11561c4
1 parent 43ce752
commit 11561c4
Show file tree

Hide file tree

Showing 24 changed files with 494 additions and 202 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -95,11 +95,11 @@ jobs:
 
       - name: Make target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: mkdir -p refined/target shared/target tensorflow/target parquet/target tools/target protobuf/target jmh/target bigquery/target avro/target scalacheck/target datastore/target neo4j/target cats/target bigtable/target guava/target project/target
+        run: mkdir -p refined/target shared/target unidocs/target tensorflow/target parquet/target site/target tools/target protobuf/target jmh/target bigquery/target avro/target scalacheck/target datastore/target neo4j/target cats/target bigtable/target guava/target project/target
 
       - name: Compress target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
-        run: tar cf targets.tar refined/target shared/target tensorflow/target parquet/target tools/target protobuf/target jmh/target bigquery/target avro/target scalacheck/target datastore/target neo4j/target cats/target bigtable/target guava/target project/target
+        run: tar cf targets.tar refined/target shared/target unidocs/target tensorflow/target parquet/target site/target tools/target protobuf/target jmh/target bigquery/target avro/target scalacheck/target datastore/target neo4j/target cats/target bigtable/target guava/target project/target
 
       - name: Upload target directories
         if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
@@ -338,3 +338,41 @@ jobs:
         env:
           JAVA_OPTS: '-Davro.version=1.8.2'
         run: sbt '++ ${{ matrix.scala }}' avro/test
+
+  site:
+    name: Generate Site
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13]
+        java: [corretto@17]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (full)
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Java (corretto@17)
+        id: setup-java-corretto-17
+        if: matrix.java == 'corretto@17'
+        uses: actions/setup-java@v4
+        with:
+          distribution: corretto
+          java-version: 17
+          cache: sbt
+
+      - name: sbt update
+        if: matrix.java == 'corretto@17' && steps.setup-java-corretto-17.outputs.cache-hit == 'false'
+        run: sbt +update
+
+      - name: Generate site
+        run: sbt '++ ${{ matrix.scala }}' site/makeSite
+
+      - name: Publish site
+        if: startsWith(github.ref, 'refs/tags/v')
+        env:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: site/target
+          keep_files: true
+        uses: peaceiris/actions-gh-pages@v3.9.3
diff --git a/README.md b/README.md
@@ -36,14 +36,7 @@ This library includes the following modules.
 
 # Usage
 
-See [derivation.md](https://github.com/spotify/magnolify/tree/master/docs/derivation.md) for type class derivation for Cats, Scalacheck, and Guava.
-
-See [avro.md](https://github.com/spotify/magnolify/tree/master/docs/avro.md)
-[bigquery.md](https://github.com/spotify/magnolify/tree/master/docs/bigquery.md)
-[bigtable.md](https://github.com/spotify/magnolify/tree/master/docs/bigtable.md)
-[datastore.md](https://github.com/spotify/magnolify/tree/master/docs/datastore.md)
-[protobuf.md](https://github.com/spotify/magnolify/tree/master/docs/protobuf.md)
-[tensorflow.md](https://github.com/spotify/magnolify/tree/master/docs/tensorflow.md) for data type conversions for these libraries. See [parquet.md](https://github.com/spotify/magnolify/tree/master/docs/parquet.md) for Parquet IO support. Also see [enums.md](https://github.com/spotify/magnolify/tree/master/docs/enums.md) for enum types and [refined.md](https://github.com/spotify/magnolify/tree/master/docs/derivation.md) for refinement types support. Finally see [mapping.md](https://github.com/spotify/magnolify/blob/master/docs/mapping.md) for a mapping table of Scala types supported by conversion and IO modules.
+See [micro-site](https://spotify.github.io/magnolify/) for documentation.
 
 # How to Release
 

diff --git a/build.sbt b/build.sbt
@@ -15,6 +15,7 @@
  */
 import sbt._
 import sbtprotoc.ProtocPlugin.ProtobufConfig
+import com.github.sbt.git.SbtGit.GitKeys.gitRemoteRepo
 import com.typesafe.tools.mima.core._
 
 val magnoliaScala2Version = "1.1.6"
@@ -105,17 +106,21 @@ val scala3 = "3.3.1"
 val scala213 = "2.13.12"
 val scala212 = "2.12.18"
 val scalaDefault = scala213
+val scala3Projects = List(
+  "shared",
+  "test"
+)
 
 // github actions
 val java17 = JavaSpec.corretto("17")
 val java11 = JavaSpec.corretto("11")
 val javaDefault = java17
 
-val scala3Cond = "matrix.scala == '3'"
-val scala3Projects = List(
-  "shared",
-  "test"
-)
+val condIsScala3 = "matrix.scala == '3'"
+val condNotScala3 = s"!($condIsScala3)"
+val condIsMain = "github.ref == 'refs/heads/main'"
+val condIsTag = "startsWith(github.ref, 'refs/tags/v')"
+
 ThisBuild / scalaVersion := scalaDefault
 ThisBuild / crossScalaVersions := Seq(scala3, scala213, scala212)
 ThisBuild / githubWorkflowTargetBranches := Seq("main")
@@ -130,19 +135,19 @@ ThisBuild / githubWorkflowBuild ~= { steps: Seq[WorkflowStep] =>
   steps.flatMap {
     case s if s.name.contains("Test") =>
       Seq(
-        s.withCond(Some(s"!($scala3Cond)")),
+        s.withCond(Some(condNotScala3)),
         WorkflowStep.Sbt(
           scala3Projects.map(p => s"$p/test"),
           name = Some("Test"),
-          cond = Some(scala3Cond)
+          cond = Some(condIsScala3)
         )
       )
     case s =>
       if (
         s.name.contains("Check binary compatibility") ||
         s.name.contains("Generate API documentation")
       ) {
-        Seq(s.withCond(Some(s"!($scala3Cond)")))
+        Seq(s.withCond(Some(condNotScala3)))
       } else {
         Seq(s)
       }
@@ -181,6 +186,35 @@ ThisBuild / githubWorkflowAddedJobs ++= Seq(
       ),
     scalas = List(CrossVersion.binaryScalaVersion(scalaDefault)),
     javas = List(javaDefault)
+  ),
+  WorkflowJob(
+    "site",
+    "Generate Site",
+    WorkflowStep.CheckoutFull ::
+      WorkflowStep.SetupJava(List(javaDefault)) :::
+      List(
+        WorkflowStep.Sbt(
+          List("site/makeSite"),
+          name = Some("Generate site")
+        ),
+        WorkflowStep.Use(
+          UseRef.Public("peaceiris", "actions-gh-pages", "v3.9.3"),
+          env = Map(
+            "github_token" -> "${{ secrets.GITHUB_TOKEN }}",
+            "publish_dir" -> {
+              val path = (ThisBuild / baseDirectory).value.toPath.toAbsolutePath
+                .relativize((site / target).value.toPath)
+              // os-independent path rendering ...
+              (0 until path.getNameCount).map(path.getName).mkString("/")
+            },
+            "keep_files" -> "true"
+          ),
+          name = Some("Publish site"),
+          cond = Some(condIsTag)
+        )
+      ),
+    scalas = List(CrossVersion.binaryScalaVersion(scalaDefault)),
+    javas = List(javaDefault)
   )
 )
 
@@ -624,3 +658,84 @@ lazy val jmh: Project = project
       "org.tensorflow" % "tensorflow-core-api" % tensorflowVersion % Test
     )
   )
+
+// =======================================================================
+// Site settings
+// =======================================================================
+lazy val site = project
+  .in(file("site"))
+  .enablePlugins(
+    ParadoxSitePlugin,
+    ParadoxMaterialThemePlugin,
+    GhpagesPlugin,
+    SiteScaladocPlugin,
+    MdocPlugin
+  )
+  .dependsOn(
+    avro % "compile->compile,provided",
+    bigquery % "compile->compile,provided",
+    bigtable % "compile->compile,provided",
+    cats % "compile->compile,provided",
+    datastore % "compile->compile,provided",
+    guava % "compile->compile,provided",
+    neo4j % "compile->compile,provided",
+    parquet % "compile->compile,provided",
+    protobuf % "compile->compile,provided",
+    refined % "compile->compile,provided",
+    shared,
+    scalacheck % "compile->compile,provided",
+    tensorflow % "compile->compile,provided",
+    unidocs
+  )
+  .settings(commonSettings)
+  .settings(
+    description := "Magnolify - Documentation",
+    fork := false,
+    publish / skip := true,
+    autoAPIMappings := true,
+    gitRemoteRepo := "git@github.com:spotify/magnolify.git",
+    // mdoc
+    // pre-compile md using mdoc
+    mdocIn := (paradox / sourceDirectory).value,
+    mdocExtraArguments ++= Seq("--no-link-hygiene"),
+    // paradox
+    Compile / paradox / sourceManaged := mdocOut.value,
+    paradoxProperties ++= Map(
+      "github.base_url" -> "https://github.com/spotify/magnolify"
+    ),
+    Compile / paradoxMaterialTheme := ParadoxMaterialTheme()
+      .withFavicon("images/favicon.ico")
+      .withColor("white", "indigo")
+      .withLogo("images/logo.png")
+      .withCopyright("Copyright (C) 2024 Spotify AB")
+      .withRepository(uri("https://github.com/spotify/magnolify"))
+      .withSocial(uri("https://github.com/spotify"), uri("https://twitter.com/spotifyeng")),
+    // sbt-site
+    addMappingsToSiteDir(
+      unidocs / ScalaUnidoc / packageDoc / mappings,
+      unidocs / ScalaUnidoc / siteSubdirName
+    ),
+    makeSite := makeSite.dependsOn(mdoc.toTask("")).value
+  )
+
+lazy val unidocs = project
+  .in(file("unidocs"))
+  .enablePlugins(TypelevelUnidocPlugin)
+  .settings(commonSettings)
+  .settings(
+    moduleName := "magnolify-docs",
+    crossScalaVersions := Seq(scalaDefault),
+    scalaVersion := scalaDefault,
+    // unidoc
+    ScalaUnidoc / siteSubdirName := "api",
+    ScalaUnidoc / scalacOptions := Seq.empty,
+    ScalaUnidoc / unidoc / unidocProjectFilter := inAnyProject -- inProjects(test, jmh),
+    ScalaUnidoc / unidoc / unidocAllClasspaths ~= { cp =>
+      // somehow protobuf 2 is in classpath and fails doc
+      cp.map(_.filterNot(_.data.getName.endsWith("protobuf-java-2.5.0.jar")))
+    },
+    ScalaUnidoc / unidoc / unidocAllSources ~= { sources =>
+      // filter out doc from generated proto TFMD sources
+      sources.map(_.filterNot(_.getPath.contains("compiled_proto")))
+    }
+  )
diff --git a/docs/derivation.md b/docs/derivation.md
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,4 +1,10 @@
-addSbtPlugin("org.typelevel" % "sbt-typelevel" % "0.6.3")
+addSbtPlugin("com.github.sbt" % "sbt-ghpages" % "0.8.0")
+addSbtPlugin("com.github.sbt" % "sbt-site-paradox" % "1.5.0")
+addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0")
+addSbtPlugin("com.lightbend.paradox" % "sbt-paradox" % "0.10.5")
 addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.6")
+addSbtPlugin("io.github.jonas" % "sbt-paradox-material-theme" % "0.6.0")
+addSbtPlugin("org.scalameta" % "sbt-mdoc" % "2.5.1")
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.9")
+addSbtPlugin("org.typelevel" % "sbt-typelevel" % "0.6.3")
 addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.6")
diff --git a/site/src/main/paradox/_template/scaladoc.st b/site/src/main/paradox/_template/scaladoc.st
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <!-- HTML meta refresh URL redirection -->
+  <meta http-equiv="refresh" content="0; url=api/magnolify/index.html">
+</head>
+<body>
+<p>The page has moved to: <a href="api/magnolify/index.html">this page</a></p>
+</body>
+</html>