From 5afc584c9338bd2b5d7dff827a04ecdd07d6ac36 Mon Sep 17 00:00:00 2001
From: Cheng Lian <lian.cs.zju@gmail.com>
Date: Sat, 9 Aug 2014 08:50:41 +0800
Subject: [PATCH] Filter out spark-submit options when starting Python gateway

---
 bin/pyspark | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/bin/pyspark b/bin/pyspark
index a655fcdd0c873..c39eb62ce5a82 100755
--- a/bin/pyspark
+++ b/bin/pyspark
@@ -23,6 +23,8 @@ FWDIR="$(cd `dirname $0`/..; pwd)"
 # Export this as SPARK_HOME
 export SPARK_HOME="$FWDIR"
 
+source $FWDIR/bin/utils.sh
+
 SCALA_VERSION=2.10
 
 if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
@@ -67,9 +69,10 @@ fi
 # We export Spark submit arguments as an environment variable because shell.py must run as a
 # PYTHONSTARTUP script, which does not take in arguments. This is required for IPython notebooks.
 
+gatherSparkSubmitOpts $@
 PYSPARK_SUBMIT_ARGS=""
 whitespace="[[:space:]]"
-for i in "$@"; do
+for i in ${SUBMISSION_OPTS[@]}; do
   if [[ $i =~ \" ]]; then i=$(echo $i | sed 's/\"/\\\"/g'); fi
   if [[ $i =~ $whitespace ]]; then i=\"$i\"; fi
   PYSPARK_SUBMIT_ARGS="$PYSPARK_SUBMIT_ARGS $i"
@@ -86,8 +89,6 @@ if [[ -n "$SPARK_TESTING" ]]; then
   exit
 fi
 
-source $FWDIR/bin/utils.sh
-
 # If a python file is provided, directly run spark-submit.
 if [[ "$1" =~ \.py$ ]]; then
   echo -e "\nWARNING: Running python applications through ./bin/pyspark is deprecated as of Spark 1.0." 1>&2