let thrift server take SPARK_DAEMON_MEMORY and SPARK_DAEMON_JAVA_OPTS

jeanlyn · Apr 21, 2015 · 4fb25ed · 4fb25ed
1 parent 1f2f723
commit 4fb25ed
Showing 1 changed file with 22 additions and 2 deletions.
diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkSubmitCommandBuilder.java
@@ -190,6 +190,10 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env) throws IOE
       firstNonEmptyValue(SparkLauncher.DRIVER_EXTRA_CLASSPATH, conf, props) : null;
 
     List<String> cmd = buildJavaCommand(extraClassPath);
+    // Take Thrift Server as daemon
+    if (isThriftServer(mainClass)) {
+      addOptionString(cmd, System.getenv("SPARK_DAEMON_JAVA_OPTS"));
+    }
     addOptionString(cmd, System.getenv("SPARK_SUBMIT_OPTS"));
     addOptionString(cmd, System.getenv("SPARK_JAVA_OPTS"));
 
@@ -201,8 +205,16 @@ private List<String> buildSparkSubmitCommand(Map<String, String> env) throws IOE
       // - SPARK_DRIVER_MEMORY env variable
       // - SPARK_MEM env variable
       // - default value (512m)
-      String memory = firstNonEmpty(firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
-        System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
+      String memory;
+      // Take Thrift Server as daemon
+      if (isThriftServer(mainClass)) {
+        memory = firstNonEmpty(System.getenv("SPARK_DAEMON_MEMORY"),
+            firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
+            System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
+      } else {
+        memory = firstNonEmpty(firstNonEmptyValue(SparkLauncher.DRIVER_MEMORY, conf, props),
+            System.getenv("SPARK_DRIVER_MEMORY"), System.getenv("SPARK_MEM"), DEFAULT_MEM);
+      }
       cmd.add("-Xms" + memory);
       cmd.add("-Xmx" + memory);
       addOptionString(cmd, firstNonEmptyValue(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, conf, props));
@@ -292,6 +304,14 @@ private boolean isClientMode(Properties userProps) {
       (!userMaster.equals("yarn-cluster") && deployMode == null);
   }
 
+  /**
+   * Return whether the given main class represents a thrift server.
+   */
+  private boolean isThriftServer(String mainClass) {
+    return mainClass.equals("org.apache.spark.sql.hive.thriftserver.HiveThriftServer2");
+  }
+
+
   private class OptionParser extends SparkSubmitOptionParser {
 
     @Override