Initial JES backend work

Dockerfile

@@ -10,7 +10,6 @@ RUN ["/bin/bash", "-c", "/cromwell/docker/install.sh /cromwell"]
 
 # Add Cromwell as a service (it will start when the container starts)
 RUN mkdir /etc/service/cromwell && \
-    cp /cromwell/src/main/resources/reference.conf /etc/cromwell.conf && \
     cp /cromwell/docker/run.sh /etc/service/cromwell/run
 
 # These next 4 commands are for enabling SSH to the container.

build.sbt

@@ -11,6 +11,7 @@ scalaVersion := "2.11.6"
 val sprayV = "1.3.2"
 val DowngradedSprayV = "1.3.1"
 val akkaV = "2.3.12"
+val googleClientApiV = "1.20.0"
 
 libraryDependencies ++= Seq(
   "com.gettyimages" %% "spray-swagger" % "0.5.1",
@@ -30,6 +31,10 @@ libraryDependencies ++= Seq(
   "com.typesafe.slick" %% "slick" % "3.0.0",
   "com.zaxxer" % "HikariCP" % "2.3.3",
   "org.hsqldb" % "hsqldb" % "2.3.2",
+  "com.google.gcloud" % "gcloud-java" % "latest.integration",
+  "com.google.api-client" % "google-api-client-java6" % googleClientApiV,
+  "com.google.api-client" % "google-api-client-jackson2" % googleClientApiV,
+  "com.google.oauth-client" % "google-oauth-client" % googleClientApiV,
   "mysql" % "mysql-connector-java" % "5.1.35",
   //---------- Test libraries -------------------//
   "io.spray" %% "spray-testkit" % sprayV % Test,

src/main/java/cromwell/parser/BackendType.java

@@ -0,0 +1,5 @@
+package cromwell.parser;
+
+public enum BackendType {
+    LOCAL, JES;
+}

src/main/java/cromwell/parser/MemorySize.java

@@ -0,0 +1,35 @@
+package cromwell.parser;
+
+/**
+ * Convenience enum for parsing memory specifications.  If feels like there should be a preexisting library
+ * for this but I couldn't find one.
+ */
+public enum MemorySize {
+    Bytes(1, "B"),
+    KiB(1_000, "KiB"),
+    MiB(KiB.multiplier * KiB.multiplier, "MiB"),
+    GiB(KiB.multiplier * MiB.multiplier, "GiB"),
+    TiB(KiB.multiplier * GiB.multiplier, "TiB"),
+    KB(1 << 10, "KB"),
+    MB(KB.multiplier * KB.multiplier, "MB"),
+    GB(KB.multiplier * MB.multiplier, "GB"),
+    TB(KB.multiplier * GB.multiplier, "TB");
+
+    public final long multiplier;
+    public final String suffix;
+
+    MemorySize(long multiplier, String suffix) {
+        this.multiplier = multiplier;
+        this.suffix = suffix;
+    }
+
+    /** Convert from the units of this memory size to individual bytes. */
+    public double toBytes(double prefix) {
+        return prefix * multiplier;
+    }
+
+    /** Convert from individual bytes to units of this memory size. */
+    public double fromBytes(double bytes) {
+        return bytes / multiplier;
+    }
+}

src/main/java/cromwell/parser/RuntimeKey.java

@@ -0,0 +1,65 @@
+package cromwell.parser;
+
+import static cromwell.parser.BackendType.JES;
+import static cromwell.parser.BackendType.LOCAL;
+
+/**
+ * Backend runtime keys and the backends which are known to support them.
+ */
+public enum RuntimeKey {
+    CPU("cpu", JES),
+    DEFAULT_DISKS("defaultDisks", JES),
+    DEFAULT_ZONES("defaultZones", JES),
+    DOCKER("docker", new BackendType[]{JES}, LOCAL),
+    FAIL_ON_STDERR("failOnStderr", LOCAL, JES),
+    MEMORY("memory", JES),
+    PREEMPTIBLE("preemptible", JES);
+
+    public final String key;
+
+    /**
+     * BackendTypes on which this key is mandatory.
+     */
+    public final BackendType[] mandatory;
+
+    /**
+     * BackendTypes optionally supporting this key.  A BackendType need only appear in one
+     * of the mandatory or optional arrays.
+     */
+    public final BackendType[] optional;
+
+    RuntimeKey(String key, BackendType... optional) {
+        this(key, new BackendType[0], optional);
+    }
+
+    RuntimeKey(String key, BackendType [] mandatory, BackendType... optional) {
+        this.key = key;
+        this.mandatory = mandatory;
+        this.optional = optional;
+    }
+
+    public boolean isMandatory(BackendType backendType) {
+        for (BackendType type : mandatory) {
+            if (type == backendType) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public boolean isOptional(BackendType backendType) {
+        for (BackendType type : optional) {
+            if (type == backendType) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Returns true if the specified key is mandatory or optional on this backend.
+     */
+    public boolean supports(BackendType backendType) {
+        return isOptional(backendType) || isMandatory(backendType);
+    }
+}

src/main/resources/application.conf

@@ -1,12 +1,88 @@
-akka {
-  event-handlers = ["akka.event.slf4j.Slf4jEventHandler"]
-  loggers = ["akka.event.slf4j.Slf4jLogger"]
-  loglevel = "DEBUG"
-  actor {
-    debug {
-      receive = on
-      event-stream = on
-      fsm = on
+webservice.port = 8000
+webservice.interface = 0.0.0.0
+instance.name = "reference"
+
+swagger {
+  apiDocs = "api-docs"
+  apiVersion = "0.1"
+  baseUrl = "/"
+  contact = "dsde@broadinstitute.org"
+  description = "One workflow service to rule them all..."
+  info = "Cromwell"
+  license = "BSD"
+  licenseUrl = "http://opensource.org/licenses/BSD-3-Clause"
+  swaggerVersion = "1.2"
+  termsOfServiceUrl = "http://www.github.com/broadinstitute/cromwell"
+}
+
+spray.can {
+  server {
+    request-timeout = 40s
+  }
+  client {
+    request-timeout = 40s
+    connecting-timeout = 40s
+  }
+}
+
+
+backend {
+  backend = "local" // Either "JES" or "local", case insensitive
+
+  // If backend is JES, must supply a 'jes' stanza:
+  //      jes
+  //          applicationName
+  //          project (i.e. google project name)
+  //          baseExecutionBucket (GCS path where things will be written to)
+}
+
+// If authenticating with Google (e.g. if backend is 'JES') you must supply a top level stanza 'google':
+
+//      google
+//           authScheme  - Either "service" for service account or "user" for user-based, case insensitive
+                // If "user" you must supply:
+                //      userAuth
+                //           user
+                //           secretsFile
+                //           dataStoreDir (dir to cache oauth credentials)
+                // You will be asked to do the oauth browser dance
+
+                // If "service" you must supply:
+                //        p12File
+                //        serviceAccountId
+
+database {
+  config = main.hsqldb
+
+  main {
+    hsqldb {
+      url = "jdbc:hsqldb:mem:${slick.uniqueSchema};shutdown=false"
+      driver = "org.hsqldb.jdbcDriver"
+      slick.driver = "slick.driver.HsqldbDriver"
+      slick.createSchema = true
+    }
+  }
+
+  test {
+    hsqldb {
+      url = "jdbc:hsqldb:mem:testdb;shutdown=false"
+      driver = "org.hsqldb.jdbcDriver"
+      slick.driver = "slick.driver.HsqldbDriver"
+      liquibase = {
+        changelog = "src/main/migrations/changelog.xml"
+        connection = "liquibase.database.jvm.HsqlConnection"
+      }
+    }
+
+    mysql {
+      url = "jdbc:mysql://localhost/cromwell_test"
+      user = "travis"
+      password = ""
+      driver = "com.mysql.jdbc.Driver"
+      slick.driver = "slick.driver.MySQLDriver"
+      liquibase = {
+        changelog = "src/main/migrations/changelog.xml"
+      }
     }
   }
 }

src/main/scala/cromwell/Main.scala

@@ -2,10 +2,13 @@ package cromwell
 
 import java.io.File
 import java.nio.file.Paths
+
+import com.typesafe.config.ConfigFactory
+import cromwell.binding._
 import cromwell.util.FileUtil.{EnhancedFile, EnhancedPath}
 import cromwell.binding.formatter.{AnsiSyntaxHighlighter, SyntaxFormatter}
 import cromwell.binding.{AstTools, _}
-import cromwell.engine.workflow.SingleWorkflowRunnerActor
+import cromwell.engine.workflow.{WorkflowManagerActor, SingleWorkflowRunnerActor}
 import cromwell.parser.WdlParser.SyntaxError
 import cromwell.server.{CromwellServer, DefaultWorkflowManagerSystem, WorkflowManagerSystem}
 import cromwell.util.FileUtil
@@ -43,14 +46,14 @@ object Main extends App {
   def validate(args: Array[String]): Unit = {
     if (args.length != 1) usageAndExit()
     try {
-      WdlNamespace.load(new File(args(0)))
+      WdlNamespace.load(new File(args(0)), WorkflowManagerActor.BackendType)
     } catch {
       case e: SyntaxError => println(e)
     }
   }
 
   def highlight(args: Array[String]) {
-    val namespace = WdlNamespace.load(new File(args(0)))
+    val namespace = WdlNamespace.load(new File(args(0)), WorkflowManagerActor.BackendType)
     val formatter = new SyntaxFormatter(AnsiSyntaxHighlighter)
     println(formatter.format(namespace))
   }
@@ -59,7 +62,7 @@ object Main extends App {
     if (args.length != 1) usageAndExit()
     try {
       import cromwell.binding.types.WdlTypeJsonFormatter._
-      val namespace = WdlNamespace.load(new File(args(0)))
+      val namespace = WdlNamespace.load(new File(args(0)), WorkflowManagerActor.BackendType)
       namespace match {
         case x: NamespaceWithWorkflow => println(x.workflow.inputs.toJson.prettyPrint)
         case _ => println("WDL does not have a local workflow")
@@ -72,6 +75,8 @@ object Main extends App {
   def run(args: Array[String], workflowManagerSystem: WorkflowManagerSystem): Unit = {
     if (args.length != 2) usageAndExit()
 
+    Log.info(s"Backend is: ${WorkflowManagerActor.BackendType}")
+
     Log.info(s"RUN sub-command")
     Log.info(s"  WDL file: ${args(0)}")
     Log.info(s"  Inputs: ${args(1)}")