Yarn support

build.sbt

@@ -72,9 +72,15 @@ libraryDependencies <+= scalaVersion {
   case _ => "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2"
 }
 
-libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.1"
+libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.1" % "provided"
+
+libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.0.3" % "provided"
+
+// Hadoop
+libraryDependencies += "org.apache.hadoop" % "hadoop-common" % "2.7.2" % "provided"
+
+libraryDependencies += "org.apache.hadoop" % "hadoop-yarn-client" % "2.7.2" % "provided"
 
-libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.0.3"
 
 // Resolvers
 
@@ -106,4 +112,3 @@ ghpages.settings
 git.remoteRepo := "git@github.com:rjagerman/glint.git"
 
 site.includeScaladoc()
-

conf/env.sh.template

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 # Change these values depending on your setup
+GLINT_JAR_NAME=Glint-0.1.jar
 GLINT_JAR_PATH=./bin/Glint-0.1.jar   # Path to the assembled glint jar file, this should be the same on all cluster machines
 GLINT_MASTER_OPTS="-Xmx2048m"        # Java options to pass the JVM when starting a master
 GLINT_SERVER_OPTS="-Xmx2048m"        # Java options to pass the JVM when starting a server

sbin/start-on-yarn.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+function usage {
+    echo "usage: -c <config file> -n <number Glint Instance> -H <hdfs://hdfs-addresss:port>"
+}
+
+CONFIG_FILE=""
+INSTANCE_NUMBER=0
+HDFS_PREFIX=""
+
+while getopts "h?:n:c:H:" arg
+do
+    case ${arg} in
+        h) usage;;
+        H) HDFS_PREFIX=${OPTARG};;
+        c) CONFIG_FILE=${OPTARG};;
+        n) INSTANCE_NUMBER=${OPTARG};;
+        \?) usage
+    esac
+done
+
+if [ ${INSTANCE_NUMBER} -lt 2 ]; then
+    echo "Glint Instance Number should be more than 2"
+    exit 1
+fi
+
+CONFIG_OPTION=""
+if [ "$CONFIG_FILE" != "" ]; then
+    CONFIG_OPTION="-c ${CONFIG_FILE}"
+fi
+
+if [ "$HDFS_PREFIX" == "" ]; then
+    echo "Not Specific HDFS Prefix"
+    exit 1
+fi
+
+# Set script path and load configuration
+GLINT_SBIN_PATH=$(dirname "$0")
+GLINT_SBIN_PATH=`cd $(dirname ${BASH_SOURCE[0]}); pwd`
+source $GLINT_SBIN_PATH/configuration.sh
+
+# Check Hadoop Conf Dir
+if [ "$HADOOP_HOME" == "" ]; then
+    echo "Without HADOOP_HOME environment var"
+    exit -1
+fi
+HADOOP_BIN="$HADOOP_HOME/bin"
+HADOOP_CMD="$HADOOP_BIN/hadoop"
+
+# Prepare Logs
+mkdir -p $GLINT_PATH/pids
+mkdir -p $GLINT_PATH/logs
+
+# Upload Jar to HDFS
+HADOOP_JAR_PATH="$HDFS_PREFIX/$GLINT_JAR_NAME"
+${HADOOP_CMD} fs -rm ${HADOOP_JAR_PATH} > $GLINT_PATH/logs/glint-on-yarn-out.log 2> $GLINT_PATH/logs/glint-on-yarn-err.log
+${HADOOP_CMD} fs -put ${GLINT_JAR_PATH} ${HADOOP_JAR_PATH} > $GLINT_PATH/logs/glint-on-yarn-out.log 2> $GLINT_PATH/logs/glint-on-yarn-err.log
+
+# Start on Yarn
+${HADOOP_CMD} jar ${GLINT_JAR_PATH} glint.yarn.AppClient ${CONFIG_OPTION} -n ${INSTANCE_NUMBER} --path ${HADOOP_JAR_PATH} > $GLINT_PATH/logs/glint-on-yarn-out.log 2> $GLINT_PATH/logs/glint-on-yarn-err.log &

src/main/resources/glint.conf

@@ -116,6 +116,43 @@ glint {
 
   }
 
+  # Configuration for Yarn Container
+  yarn {
+    # Memory for AM
+    am-memory = 1024
+
+    # vCore for AM
+    am-vcores = 2
+
+    # Memory OverHead for AM
+    am-memory-overhead = 1024
+
+    # Vcore for container
+    container-vcores = 2
+
+    # Memory for container
+    container-memory = 10240
+
+    # OverHead Memory for container
+    container-memory-overhead = 2048
+
+    # Minium Glint server number
+    minimum-server-number = 10
+
+    # Monitor Interval
+    monitor-interval = 60
+
+    # Number Of Max Container Failure
+    max-number-container-failures = 10
+
+    # HeartBeat Interval
+    heartbeat-interval = 10
+
+    # Initial Allcation Interval
+    initial-allocation-interval = 60
+  }
+
+
   # Default configuration setting
   default {