support spark3

WeDataSphere · Jan 17, 2025 · 461fba2 · 461fba2
1 parent 7186259
commit 461fba2
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 6 deletions.
diff --git a/...nkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala b/...nkis-entrance/src/main/scala/org/apache/linkis/entrance/parser/CommonEntranceParser.scala
@@ -35,10 +35,15 @@ import org.apache.linkis.manager.label.conf.LabelCommonConfig
 import org.apache.linkis.manager.label.constant.LabelKeyConstant
 import org.apache.linkis.manager.label.entity.Label
 import org.apache.linkis.manager.label.entity.cluster.ClusterLabel
-import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, UserCreatorLabel}
+import org.apache.linkis.manager.label.entity.engine.{
+  CodeLanguageLabel,
+  EngineType,
+  UserCreatorLabel
+}
 import org.apache.linkis.manager.label.utils.EngineTypeLabelCreator
 import org.apache.linkis.protocol.constants.TaskConstant
 import org.apache.linkis.scheduler.queue.SchedulerEventState
+import org.apache.linkis.storage.script.VariableParser
 
 import org.apache.commons.lang3.StringUtils
 
@@ -265,9 +270,17 @@ class CommonEntranceParser(val persistenceManager: PersistenceManager)
       if (formatCode) executionCode = format(executionCode)
       jobReq.setExecutionCode(executionCode)
     }
-    val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(executeApplicationName)
+    var engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(executeApplicationName)
     val runTypeLabel =
       labelBuilderFactory.createLabel[Label[_]](LabelKeyConstant.CODE_TYPE_KEY, runType)
+    val variableMap =
+      jobReq.getParams.get(VariableParser.VARIABLE).asInstanceOf[util.Map[String, String]]
+    if (variableMap.containsKey(LabelCommonConfig.SPARK3_ENGINE_VERSION)) {
+      engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(
+        EngineType.SPARK.toString,
+        variableMap.get(LabelCommonConfig.SPARK3_ENGINE_VERSION)
+      )
+    }
     val userCreatorLabel = labelBuilderFactory
       .createLabel[Label[_]](LabelKeyConstant.USER_CREATOR_TYPE_KEY, umUser + "-" + creator)
 

diff --git a/...is-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java b/...is-label-common/src/main/java/org/apache/linkis/manager/label/conf/LabelCommonConfig.java
@@ -35,7 +35,7 @@ public class LabelCommonConfig {
 
   public static final CommonVars<String> SPARK_ENGINE_VERSION =
       CommonVars.apply("wds.linkis.spark.engine.version", "2.4.3");
-
+  public static final String SPARK3_ENGINE_VERSION = "linkis.spark3.engine.version";
   public static final CommonVars<String> HIVE_ENGINE_VERSION =
       CommonVars.apply("wds.linkis.hive.engine.version", "2.3.3");
 

diff --git a/...el-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java b/...el-common/src/main/java/org/apache/linkis/manager/label/utils/EngineTypeLabelCreator.java
@@ -123,4 +123,18 @@ public static void registerVersion(String type, String version) {
     }
     defaultVersion.put(type, version);
   }
+
+  public static EngineTypeLabel createEngineTypeLabel(String type, String version) {
+    if (null == defaultVersion) {
+      init();
+    }
+    EngineTypeLabel label = labelBuilderFactory.createLabel(EngineTypeLabel.class);
+    label.setEngineType(type);
+    if (StringUtils.isNotBlank(version)) {
+      label.setVersion(version);
+    } else {
+      defaultVersion.get(type);
+    }
+    return label;
+  }
 }
diff --git a/linkis-engineconn-plugins/spark/pom.xml b/linkis-engineconn-plugins/spark/pom.xml
@@ -485,6 +485,15 @@
     </plugins>
   </build>
   <profiles>
+    <profile>
+      <id>spark-3.4</id>
+      <properties>
+        <spark.version>3.4.4</spark.version>
+        <scala.version>2.12.17</scala.version>
+        <scala.binary.version>2.12</scala.binary.version>
+        <jackson-bom.version>2.14.2</jackson-bom.version>
+      </properties>
+    </profile>
     <!-- spark2-hadoop3 version:spark2.4 use hadoop2.7.2 by default mvn validate -Pspark-2.4-hadoop-3.3  -->
     <profile>
       <id>spark-2.4-hadoop-3.3</id>

diff --git a/pom.xml b/pom.xml
@@ -102,7 +102,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <revision>1.10.0-wds</revision>
+    <revision>1.10.0-wds-spark</revision>
     <jedis.version>2.9.2</jedis.version>
     <spark.version>2.4.3</spark.version>
     <hadoop.version>2.7.2</hadoop.version>
@@ -192,6 +192,7 @@
     <!-- dev env -->
     <java.version>1.8</java.version>
     <maven.version>3.5.0</maven.version>
+    <json4s.version>3.7.0-M11</json4s.version>
     <scala.version>2.11.12</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
     <ant.version>1.10.12</ant.version>
@@ -1490,9 +1491,11 @@
     <profile>
       <id>spark-3.2</id>
       <properties>
-        <spark.version>3.2.1</spark.version>
-        <scala.version>2.12.15</scala.version>
+        <json4s.version>3.7.0-M11</json4s.version>
+        <spark.version>3.4.4</spark.version>
+        <scala.version>2.12.17</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
+        <jackson-bom.version>2.14.2</jackson-bom.version>
       </properties>
     </profile>