Add Spark 3.1.2 / 3.2.0 crossbuild

typelevel · Oct 18, 2021 · 1911a94 · 1911a94
1 parent c57c4f7
commit 1911a94
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -24,6 +24,7 @@ jobs:
         os: [ubuntu-latest]
         scala: [2.12.15]
         java: [adopt@1.8]
+        sparkVersion: [3.2.0, 3.1.2]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout current branch (full)
@@ -60,12 +61,16 @@ jobs:
         run: pip install codecov
 
       - name: Test & Compute Coverage
+        env:
+          SPARK_VERSION: ${{ matrix.sparkVersion }}
         run: sbt ++${{ matrix.scala }} coverage test coverageReport
 
       - name: Upload Codecov Results
         run: codecov -F ${{ matrix.scala }}
 
       - name: Binary compatibility check
+        env:
+          SPARK_VERSION: ${{ matrix.sparkVersion }}
         run: sbt ++${{ matrix.scala }} mimaReportBinaryIssues
 
   publish:

diff --git a/build.sbt b/build.sbt
@@ -1,4 +1,4 @@
-val sparkVersion = "3.2.0"
+val sparkVersion = sys.env.get("SPARK_VERSION").getOrElse("3.2.0")
 val catsCoreVersion = "2.6.1"
 val catsEffectVersion = "2.4.0"
 val catsMtlVersion = "0.7.1"
@@ -256,25 +256,33 @@ copyReadme := copyReadmeImpl.value
 
 ThisBuild / githubWorkflowArtifactUpload := false
 
+ThisBuild / githubWorkflowBuildMatrixAdditions := Map("sparkVersion" -> List("3.2.0", "3.1.2"))
+
 ThisBuild / githubWorkflowBuild := Seq(
-  WorkflowStep.Use(UseRef.Public("actions", "setup-python", "v2"),
-                   name = Some("Setup Python"),
-                   params = Map("python-version" -> "3.x")
+  WorkflowStep.Use(
+    UseRef.Public("actions", "setup-python", "v2"),
+    name = Some("Setup Python"),
+    params = Map("python-version" -> "3.x")
   ),
-  WorkflowStep.Run(List("pip install codecov"),
-                   name = Some("Setup codecov")
+  WorkflowStep.Run(
+    List("pip install codecov"),
+    name = Some("Setup codecov")
   ),
-  WorkflowStep.Sbt(List("coverage", "test", "coverageReport"),
-                   name = Some("Test & Compute Coverage")
+  WorkflowStep.Sbt(
+    List("coverage", "test", "coverageReport"),
+    name = Some("Test & Compute Coverage"),
+    env  = Map("SPARK_VERSION" -> "${{ matrix.sparkVersion }}")
   ),
-  WorkflowStep.Run(List("codecov -F ${{ matrix.scala }}"),
-                   name = Some("Upload Codecov Results")
+  WorkflowStep.Run(
+    List("codecov -F ${{ matrix.scala }}"),
+    name = Some("Upload Codecov Results")
   )
 )
 
 ThisBuild / githubWorkflowBuild += WorkflowStep.Sbt(
   List("mimaReportBinaryIssues"),
-  name = Some("Binary compatibility check")
+  name = Some("Binary compatibility check"),
+  env  = Map("SPARK_VERSION" -> "${{ matrix.sparkVersion }}")
 )
 
 ThisBuild / githubWorkflowPublishTargetBranches := Seq(
@@ -301,8 +309,9 @@ ThisBuild / githubWorkflowAddedJobs ++= Seq(
     "docs",
     "Documentation",
     githubWorkflowJobSetup.value.toList ::: List(
-      WorkflowStep.Sbt(List("doc", "mdoc"),
-                       name = Some("Documentation")
+      WorkflowStep.Sbt(
+        List("doc", "mdoc"),
+        name = Some("Documentation")
       )
     ),
     scalas = List(Scala212)

diff --git a/dataset/src/main/scala/org/apache/spark/sql/FramelessInternals.scala b/dataset/src/main/scala/org/apache/spark/sql/FramelessInternals.scala
@@ -59,7 +59,7 @@ object FramelessInternals {
     def dataType: DataType = tagged.dataType
     protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = ???
     override def genCode(ctx: CodegenContext): ExprCode = tagged.genCode(ctx)
-    override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)
+    protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)
   }
 
   /** Expression to tag columns from the right hand side of join expression. */
@@ -70,6 +70,6 @@ object FramelessInternals {
     def dataType: DataType = tagged.dataType
     protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = ???
     override def genCode(ctx: CodegenContext): ExprCode = tagged.genCode(ctx)
-    override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)
+    protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = copy(newChildren.head)
   }
 }