[NSE-1191] fix AQE exchange reuse in Spark3.2 (oap-project#1192)

Fix AQE exchange reuse in Spark3.2 Note with this patch, the jars for Spark321 and Spark322 are separated. - to build jar for Spark322 use -Pspark-3.2.2 - to build jar for Spark321 use -Pspark3.2.1 Signed-off-by: Yuan Zhou yuan.zhou@intel.com
zhouyuan · Dec 16, 2022 · 41e1283 · 41e1283
1 parent 16e5042
commit 41e1283
Show file tree

Hide file tree

Showing 10 changed files with 1,794 additions and 3 deletions.
diff --git a/gazelle-dist/pom.xml b/gazelle-dist/pom.xml
@@ -55,6 +55,36 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>spark-3.2.1</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark32</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark321</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>spark-3.2.2</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark32</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark322</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+      </dependencies>
+    </profile>
   </profiles>
 
   <dependencies>

diff --git a/native-sql-engine/core/pom.xml b/native-sql-engine/core/pom.xml
@@ -78,6 +78,40 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>spark-3.2.1</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark32</artifactId>
+          <version>${project.version}</version>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark321</artifactId>
+          <version>${project.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
+    <profile>
+      <id>spark-3.2.2</id>
+      <dependencies>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark32</artifactId>
+          <version>${project.version}</version>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>com.intel.oap</groupId>
+          <artifactId>spark-sql-columnar-shims-spark322</artifactId>
+          <version>${project.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
   </profiles>
 
   <dependencies>

diff --git a/pom.xml b/pom.xml
@@ -58,6 +58,26 @@
         <maven.test.skip>true</maven.test.skip>
       </properties>
     </profile>
+    <profile>
+      <id>spark-3.2.1</id>
+      <properties>
+        <spark.version>${spark321.version}</spark.version>
+        <scala.version>2.12.15</scala.version>
+        <!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
+        <jackson.version>2.12.0</jackson.version>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
+    <profile>
+      <id>spark-3.2.2</id>
+      <properties>
+        <spark.version>${spark322.version}</spark.version>
+        <scala.version>2.12.15</scala.version>
+        <!--Jackson may be directly used in future UT. Align with the version in spark 3.2.-->
+        <jackson.version>2.12.0</jackson.version>
+        <maven.test.skip>true</maven.test.skip>
+      </properties>
+    </profile>
     <profile>
       <id>hadoop-2.7.4</id>
       <properties>
@@ -128,6 +148,7 @@
     <spark.version>3.1.1</spark.version>
     <spark311.version>3.1.1</spark311.version>
     <spark321.version>3.2.1</spark321.version>
+    <spark322.version>3.2.2</spark322.version>
     <!-- Scala 2.12.10 is the version for default spark 3.1 -->
     <scala.version>2.12.10</scala.version>
     <java.version>1.8</java.version>

diff --git a/shims/pom.xml b/shims/pom.xml
@@ -80,6 +80,26 @@
         <module>spark321</module>
       </modules>
     </profile>
+    <profile>
+      <id>spark-3.2.1</id>
+      <properties>
+      </properties>
+      <modules>
+        <module>common</module>
+        <module>spark32</module>
+        <module>spark321</module>
+      </modules>
+    </profile>
+    <profile>
+      <id>spark-3.2.2</id>
+      <properties>
+      </properties>
+      <modules>
+        <module>common</module>
+        <module>spark32</module>
+        <module>spark322</module>
+      </modules>
+    </profile>
   </profiles>
 
   <build>

diff --git a/shims/spark321/src/main/scala/com/intel/oap/sql/shims/spark321/SparkShimProvider.scala b/shims/spark321/src/main/scala/com/intel/oap/sql/shims/spark321/SparkShimProvider.scala
@@ -20,9 +20,7 @@ import com.intel.oap.sql.shims.{SparkShims, SparkShimDescriptor}
 
 object SparkShimProvider {
   val DESCRIPTOR = SparkShimDescriptor(3, 2, 1)
-  val DESCRIPTOR_SPARK321 = SparkShimDescriptor(3, 2, 1)
-  val DESCRIPTOR_SPARK322 = SparkShimDescriptor(3, 2, 2)
-  val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR_SPARK321", s"$DESCRIPTOR_SPARK322")
+  val DESCRIPTOR_STRINGS = Seq(s"$DESCRIPTOR")
 }
 
 class SparkShimProvider extends com.intel.oap.sql.shims.SparkShimProvider {