[WorkInProgress] [#247] [spark] Bump Spark Version to 2.4.7

.travis.yml

@@ -29,4 +29,4 @@ before_install:
 
 # Build
 install: 
-- build/gimel -Dmaven.test.skip=true -pl gimel-dataapi/gimel-tools -am | egrep -v "Download|Copy|Including|WARNING"
+- sh -x build/gimel -Dmaven.test.skip=true -pl gimel-dataapi | egrep -iv "Download|Copy|Including|WARNING" 

build/gimel

@@ -15,9 +15,81 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+echo "Present Working Dir"
+echo $PWD
+
 this_dir=`dirname build/install_dependencies`
-# Source reusable functions
-source ${this_dir}/gimel_functions
+
+#----------------------------function will check for error code & exit if failure, else proceed further----------------------------#
+
+#usage : check_error <$?> <custom_error_code>
+#Example: Check_error < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+check_error()
+{
+	cmd_error_code=$1
+	custom_message=$2
+	if  [ ${cmd_error_code} -gt 0 ]; then
+	  write_log "Error    | Stage |  ${custom_message}"
+	  exit ${cmd_error_code}
+	else
+	  write_log "Success  | Stage | ${custom_message}"
+	fi
+}
+
+#----------------------------function will check for error code & warn if failure----------------------------#
+
+#usage : check_warning <$?> <custom_error_code>
+#Example: Check_warning < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+
+check_warning()
+{
+
+        cmd_error_code=$1
+        pgm_exit_code=$2
+        pgm_exit_msg=$3
+        if  [ ${cmd_error_code} -gt 0 ]; then
+                write_log "WARNING ! ${cmd_error_code} ${pgm_exit_code} ${pgm_exit_msg}"
+        else
+                echo ""
+        fi
+}
+
+
+
+#----------------------------function will write the message to Console / Log File----------------------------#
+
+#Usage : write_log < Whatever message you need to log >
+
+write_log()
+{
+        msg=$1
+        to_be_logged="$(date '+%Y%m%d %H:%M:%S') | $msg"
+        echo ${to_be_logged}
+}
+
+#-----------------------------------Executes a Command--------------------------------------------------------#
+
+
+
+#Usage : run_cmd < The command to execute > 
+
+run_cmd()
+{
+       cmd=$1
+       if [ -z $2 ]; then 
+         fail_on_error="break_code"
+       else
+         fail_on_error=$2
+       fi 
+       write_log "Executing Command --> $1"
+       $cmd
+       error_code=$?
+       if [ ! $fail_on_error = "ignore_errors" ]; then
+           check_error $error_code "$cmd"
+       fi
+}
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 
@@ -34,9 +106,29 @@ else
 fi
 
 
-build/install_dependencies
-check_error $? "build/install_dependencies"
+write_log "Installing dependencies [sh -x build/install_dependencies]"
+
+# All jars below are not present in maven central or any public repository, thats why they are added manually while building gimel.
+
+mvn install:install-file -DgroupId=qubole-hive-JDBC -DartifactId=qubole-hive-JDBC -Dversion=0.0.7 -Dpackaging=jar -Dfile=${this_dir}/../lib/qubole-hive-JDBC.jar 1>>/dev/null 2>&1
+check_error $? "install qubole-hive-JDBC"
+
+mvn install:install-file -DgroupId=com.hortonworks -DartifactId=shc-core -Dversion=1.1.2-2.3-s_2.11 -Dpackaging=jar -Dfile=${this_dir}/../lib/shc-core.jar 1>>/dev/null 2>&1
+check_error $? "install shc-core"
+
+mvn install:install-file -DgroupId=com.osscube -DartifactId=aerospike-spark -Dversion=0.3-SNAPSHOT -Dpackaging=jar -Dfile=${this_dir}/../lib/aerospike-spark.jar 1>>/dev/null 2>&1
+check_error $? "install aerospike-spark"
+
+mvn install:install-file -DgroupId=com.twitter -DartifactId=zookeeper-client_2.10 -Dversion=2.0.0_fs-b -Dpackaging=jar -Dfile=${this_dir}/../lib/zookeeper-client_2.10-2.0.0_fs-b.jar 1>>/dev/null 2>&1
+check_error $? "install zookeeper-client"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=terajdbc4 -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/terajdbc4-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install terajdbc4"
+
+mvn install:install-file -DgroupId=com.teradata.jdbc -DartifactId=tdgssconfig -Dversion=15.10.00.22 -Dpackaging=jar -Dfile=${this_dir}/../lib/tdgssconfig-15.10.00.22.jar 1>>/dev/null 2>&1
+check_error $? "install tdgssconfig"
 
+write_log "Building the project [mvn install ${user_args}]"
 mvn install "$@"
 check_error $? "mvn install $@"
 

build/install_dependencies

@@ -18,8 +18,77 @@
 
 this_script=`pwd`/$BASH_SOURCE
 this_dir=`dirname $this_script`
-# Source reusable functions
-source ${this_dir}/gimel_functions
+
+#----------------------------function will check for error code & exit if failure, else proceed further----------------------------#
+
+#usage : check_error <$?> <custom_error_code>
+#Example: Check_error < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+check_error()
+{
+	cmd_error_code=$1
+	custom_message=$2
+	if  [ ${cmd_error_code} -gt 0 ]; then
+	  write_log "Error    | Stage |  ${custom_message}"
+	  exit ${cmd_error_code}
+	else
+	  write_log "Success  | Stage | ${custom_message}"
+	fi
+}
+
+#----------------------------function will check for error code & warn if failure----------------------------#
+
+#usage : check_warning <$?> <custom_error_code>
+#Example: Check_warning < pass $? from the shell command > < Custom Message for errorcode -gt 0 >
+
+
+check_warning()
+{
+
+        cmd_error_code=$1
+        pgm_exit_code=$2
+        pgm_exit_msg=$3
+        if  [ ${cmd_error_code} -gt 0 ]; then
+                write_log "WARNING ! ${cmd_error_code} ${pgm_exit_code} ${pgm_exit_msg}"
+        else
+                echo ""
+        fi
+}
+
+
+
+#----------------------------function will write the message to Console / Log File----------------------------#
+
+#Usage : write_log < Whatever message you need to log >
+
+write_log()
+{
+        msg=$1
+        to_be_logged="$(date '+%Y%m%d %H:%M:%S') | $msg"
+        echo ${to_be_logged}
+}
+
+#-----------------------------------Executes a Command--------------------------------------------------------#
+
+
+
+#Usage : run_cmd < The command to execute > 
+
+run_cmd()
+{
+       cmd=$1
+       if [ -z $2 ]; then 
+         fail_on_error="break_code"
+       else
+         fail_on_error=$2
+       fi 
+       write_log "Executing Command --> $1"
+       $cmd
+       error_code=$?
+       if [ ! $fail_on_error = "ignore_errors" ]; then
+           check_error $error_code "$cmd"
+       fi
+}
 
 #--------------------------------Begin execution of Steps------------------------------------------------#
 

docs/getting-started/gimel-modules.md

@@ -14,7 +14,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-tools</artifactId> <!--Refer one of the below listed 3 versions, depending on the required spark version -->
-      <version>2.0.0-SNAPSHOT</version> <!--provides spark 2.2.0 compiled code-->
+      <version>2.4.7-SNAPSHOT</version> <!--provides spark 2.4.7 compiled code-->
       <scope>provided</scope> <!--Ensure scope is provided as the gimel libraries can be added at runtime-->
     </dependency>
 ```
@@ -23,7 +23,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-sql</artifactId> <!--Refer one of the below listed 3 versions, depending on the required spark version -->
-      <version>2.0.0-SNAPSHOT</version> <!--provides spark 2.2.0 compiled code-->
+      <version>2.4.7-SNAPSHOT</version> <!--provides spark 2.4.7 compiled code-->
       <scope>provided</scope> <!--Ensure scope is provided as the gimel libraries can be added at runtime-->
     </dependency>
 ```
@@ -32,7 +32,7 @@
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-core</artifactId> <!--Refer one of the below listed 3 versions, depending on the required spark version -->
-      <version>2.0.0-SNAPSHOT</version> <!--provides spark 2.2.0 compiled code-->
+      <version>2.4.7-SNAPSHOT</version> <!--provides spark 2.4.7 compiled code-->
       <scope>provided</scope> <!--Ensure scope is provided as the gimel libraries can be added at runtime-->
     </dependency>
 ```

docs/gimel-connectors/kafka2.md

@@ -117,7 +117,7 @@
 
 ```bash
 spark-shell --jars 
-gimel-tools-2.0.0-SNAPSHOT-uber.jar,
+gimel-tools-2.4.7-SNAPSHOT-uber.jar,
 generic-deserializers-1.0-SNAPSHOT-uber.jar,
 generic-serializers-1.0-SNAPSHOT-uber.jar
 
@@ -510,7 +510,7 @@ Users can implement their own logic for getting the kafka properties which will
      <dependency>
         <groupId>com.paypal.gimel</groupId>
         <artifactId>gimel-common</artifactId>
-        <version>2.0.0-SNAPSHOT</version>
+        <version>2.4.7-SNAPSHOT</version>
         <scope>provided</scope>
      </dependency>
     ```

docs/gimel-logging/gimel-logging.md

@@ -75,7 +75,7 @@ object SparkWordCount {
 Example:
 
 ```shell script
-spark-shell --jars gimel-tools-2.0.0-SNAPSHOT-uber.jar \
+spark-shell --jars gimel-tools-2.4.7-SNAPSHOT-uber.jar \
 --conf spark.driver.extraJavaOptions="-Dgimel.logger.properties.filepath=gimelLoggerConfig.properties" \
 --conf spark.executor.extraJavaOptions="-Dgimel.logger.properties.filepath=gimelLoggerConfig.properties" \
 --conf spark.files=/path/to/gimelLoggerConfig.properties \

docs/gimel-serde/gimel-serde.md

@@ -57,7 +57,7 @@ Example:
 spark-shell --jars \
 generic-deserializers-1.0-SNAPSHOT-uber.jar,\
 generic-serializers-1.0-SNAPSHOT-uber.jar,\
-gimel-tools-2.0.0-SNAPSHOT-uber.jar
+gimel-tools-2.4.7-SNAPSHOT-uber.jar
 ```
 
 ## Generic Deserializers

docs/try-gimel/0-prerequisite.md

@@ -89,7 +89,7 @@ quickstart/start-gimel kafka,elasticsearch,hbase-master,hbase-regionserver
 ```
 docker exec -it spark-master bash -c \
 "export USER=an;export SPARK_HOME=/spark/;export PATH=$PATH:$SPARK_HOME/bin:$SPARK_HOME/sbin; \
-/spark/bin/spark-shell --jars /root/gimel-sql-2.0.0-SNAPSHOT-uber.jar"
+/spark/bin/spark-shell --jars /root/gimel-sql-2.4.7-SNAPSHOT-uber.jar"
 ```
 
 **Note:** *You can view the Spark UI  <a href="http://localhost:4040" target="_blank">here</a>*

gimel-dataapi/gimel-common/pom.xml

@@ -23,13 +23,13 @@ under the License.
   <parent>
     <artifactId>gimel-dataapi</artifactId>
     <groupId>com.paypal.gimel</groupId>
-   <version>2.0.0-SNAPSHOT</version>
+   <version>2.4.7-SNAPSHOT</version>
     <relativePath>../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>gimel-common</artifactId>
-  <version>2.0.0-SNAPSHOT</version>
+  <version>2.4.7-SNAPSHOT</version>
 
   <dependencies>
     <dependency>
@@ -55,9 +55,9 @@ under the License.
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-xml</artifactId>
-      <version>2.11.0-M4</version>
+      <groupId>org.scala-lang.modules</groupId>
+      <artifactId>scala-xml_${scala.binary.version}</artifactId>
+      <version>${scala.xml.version}</version>
       <scope>${scala.packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -82,12 +82,6 @@ under the License.
       <groupId>com.paypal.gimel</groupId>
       <artifactId>gimel-logger</artifactId>
       <version>${gimel.version}-SNAPSHOT</version>
-      <exclusions>
-        <exclusion>
-          <groupId>org.apache.kafka</groupId>
-          <artifactId>kafka-clients</artifactId>
-        </exclusion>
-      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.apache.kafka</groupId>
@@ -97,7 +91,7 @@ under the License.
     <dependency>
       <groupId>com.paypal.gimel</groupId>
       <artifactId>serde-common</artifactId>
-      <version>1.0-SNAPSHOT</version>
+      <version>2.4.7-SNAPSHOT</version>
       <scope>${packaging.scope}</scope>
     </dependency>
     <dependency>
@@ -106,12 +100,6 @@ under the License.
       <version>${kafka.version}</version>
       <scope>${packaging.scope}</scope>
     </dependency>
-    <dependency>
-      <groupId>com.databricks</groupId>
-      <artifactId>spark-avro_${scala.binary.version}</artifactId>
-      <version>4.0.0</version>
-      <scope>${packaging.scope}</scope>
-    </dependency>
     <dependency>
       <groupId>org.apache.curator</groupId>
       <artifactId>curator-framework</artifactId>
@@ -162,7 +150,7 @@ under the License.
     </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-streaming-kafka-${kafka.binary.version}_${scala.binary.version}</artifactId>
+      <artifactId>spark-streaming-kafka-${spark.kafka.connector.version}</artifactId>
       <version>${spark.version}</version>
       <scope>${spark.packaging.scope}</scope>
     </dependency>
@@ -197,7 +185,7 @@ under the License.
       <version>${confluent.version}</version>
       <scope>test</scope>
     </dependency>
-    <!-- Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
+<!--     Kafka local testing utility needs Netty 3.x at test scope for the minicluster -->
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty</artifactId>
@@ -211,7 +199,7 @@ under the License.
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId> net.jpountz.lz4</groupId>
+      <groupId>net.jpountz.lz4</groupId>
       <artifactId>lz4</artifactId>
       <version>1.3.0</version>
       <scope>test</scope>
@@ -220,16 +208,19 @@ under the License.
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-core</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-annotations</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
       <version>${jackson.version}</version>
+      <scope>compile</scope>
     </dependency>
   </dependencies>
 
@@ -277,7 +268,7 @@ under the License.
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-shade-plugin</artifactId>
-        <version>3.2.1</version>
+        <version>${maven.shade.plugin.version}</version>
         <configuration>
           <relocations>
             <relocation>