adding cromwell example backends folder (#4697)

* adding cromwell example backends folder

* add list to config so user can easily see

* adding note that reference.conf will ensure working Cromwell

Signed-off-by: Vanessa Sochat <vsochat@stanford.edu>

* changing script to docker_script and removing out and err for HPC (udocker)

* fixing links to develop branch and updated file in docs

cromwell.example.backends/AWS.conf

@@ -0,0 +1,24 @@
+# This is an example of how you can use the Amazon Web Services Backend
+# provider. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/tutorials/AwsBatch101/
+
+backend {
+  default = AWS
+
+  providers {
+    AWS {
+      actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory"
+      config {
+        ## These two settings are required to authenticate with the ECS service:
+        accessKeyId = "..."
+        secretKey = "..."
+      }
+    }
+}

cromwell.example.backends/BCS.conf

@@ -0,0 +1,56 @@
+# This is an example of how you can use the Alibaba Cloud Batch Compute (BCS)
+# backend provider. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/backends/BCS/
+
+backend {
+  default = BCS
+
+  providers {
+    BCS {
+      actor-factory = "cromwell.backend.impl.bcs.BcsBackendLifecycleActorFactory"
+      config {
+        root = "oss://your-bucket/cromwell-exe"
+        dockerRoot = "/cromwell-executions"
+        region = ""
+
+        #access-id = ""
+        #access-key = ""
+	#	#security-token = ""
+
+        filesystems {
+          oss {
+            auth {
+                #endpoint = ""
+                #access-id = ""
+                #access-key = ""
+		#security-token = ""
+            }
+          }
+        }
+
+        default-runtime-attributes {
+            #failOnStderr: false
+            #continueOnReturnCode: 0
+            #cluster: "cls-mycluster"
+            #mounts: "oss://bcs-bucket/bcs-dir/ /home/inputs/ false"
+            #docker: "ubuntu/latest oss://bcs-reg/ubuntu/"
+            #userData: "key value"
+            #reserveOnFail: true
+            #autoReleaseJob: true
+            #verbose: false
+            #workerPath: "oss://bcs-bucket/workflow/worker.tar.gz"
+            #systemDisk: "cloud 50"
+            #dataDisk: "cloud 250 /home/data/"
+            #timeout: 3000
+            #vpc: "192.168.0.0/16 vpc-xxxx"
+        }
+      }
+    }
+}

cromwell.example.backends/Docker.conf

@@ -0,0 +1,28 @@
+# This is an example of how you can use Docker only workflows as a Cromwell
+# backend provider. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# This backend doesn't have an official page, but you can read about general
+# Docker use here: https://cromwell.readthedocs.io/en/develop/tutorials/Containers/#docker
+# If you want to use containers, the other sections on that page will be useful to you.
+
+backend {
+  default = Docker
+
+  providers {
+
+    # Example backend that _only_ runs workflows that specify docker for every command.
+    Docker {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        run-in-background = true
+        runtime-attributes = "String docker"
+        submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash < ${docker_script}"
+      }
+    }
+}

cromwell.example.backends/HtCondor.conf

@@ -0,0 +1,84 @@
+# This is an example of how you can use the the HtCondor backend with Cromwell.
+# *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/backends/HTcondor/
+
+backend {
+  default = HtCondor
+
+  providers {
+    HtCondor {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        runtime-attributes = """
+          Int cpu = 1
+          Float memory_mb = 512.0
+          Float disk_kb = 256000.0
+          String? nativeSpecs
+          String? docker
+        """
+
+        # If an 'exit-code-timeout-seconds' value is specified:
+        # - When a job has not been alive for longer than this timeout
+        # - And has still not produced an RC file
+        # - Then it will be marked as Failed.
+        # Warning: If set, Cromwell has to run 'check-alive' for every job at regular intervals (unrelated to this timeout)
+
+        # exit-code-timeout-seconds = 120
+
+        submit = """
+          chmod 755 ${script}
+          cat > ${cwd}/execution/submitFile <<EOF
+          Iwd=${cwd}/execution
+          requirements=${nativeSpecs}
+          leave_in_queue=true
+          request_memory=${memory_mb}
+          request_disk=${disk_kb}
+          error=${err}
+          output=${out}
+          log_xml=true
+          request_cpus=${cpu}
+          executable=${script}
+          log=${cwd}/execution/execution.log
+          queue
+          EOF
+          condor_submit ${cwd}/execution/submitFile
+        """
+
+        submit-docker = """
+          chmod 755 ${script}
+          cat > ${cwd}/execution/dockerScript <<EOF
+          #!/bin/bash
+          docker run --rm -i -v ${cwd}:${docker_cwd} ${docker} /bin/bash ${script}
+          EOF
+          chmod 755 ${cwd}/execution/dockerScript
+          cat > ${cwd}/execution/submitFile <<EOF
+          Iwd=${cwd}/execution
+          requirements=${nativeSpecs}
+          leave_in_queue=true
+          request_memory=${memory_mb}
+          request_disk=${disk_kb}
+          error=${cwd}/execution/stderr
+          output=${cwd}/execution/stdout
+          log_xml=true
+          request_cpus=${cpu}
+          executable=${cwd}/execution/dockerScript
+          log=${cwd}/execution/execution.log
+          queue
+          EOF
+          condor_submit ${cwd}/execution/submitFile
+        """
+
+        kill = "condor_rm ${job_id}"
+        check-alive = "condor_q ${job_id}"
+        job-id-regex = "(?sm).*cluster (\\d+)..*"
+      }
+    }
+}
+

cromwell.example.backends/LSF.conf

@@ -0,0 +1,25 @@
+# This is an example of how you can use the LSF platform backend
+# with Cromwell. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/backends/LSF/
+
+backend {
+  default = LSF
+
+  providers {
+    LSF {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        submit = "bsub -J ${job_name} -cwd ${cwd} -o ${out} -e ${err} /usr/bin/env bash ${script}"
+        kill = "bkill ${job_id}"
+        check-alive = "bjobs ${job_id}"
+        job-id-regex = "Job <(\\d+)>.*"
+      }
+    }
+}

cromwell.example.backends/LocalExample.conf

@@ -0,0 +1,117 @@
+# This is an example of how you can use the LocalExample backend to define
+# a new backend provider. *This is not a complete configuration file!* The
+# content here should be copy pasted into the backend -> providers section
+# of the cromwell.examples.conf in the root of the repository. You should
+# uncomment lines that you want to define, and read carefully to customize
+# the file. If you have any questions, please open an issue at
+# https://www.github.com/broadinstitute/cromwell/issues
+
+# Documentation
+# https://cromwell.readthedocs.io/en/stable/backends/Local/
+
+    # Define a new backend provider.
+
+    LocalExample {
+
+      # The actor that runs the backend. In this case, it's the Shared File System (SFS) ConfigBackend.
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+
+      # The backend custom configuration.
+      config {
+
+        # Optional limits on the number of concurrent jobs
+        #concurrent-job-limit = 5
+
+        # If true submits scripts to the bash background using "&". Only usefull for dispatchers that do NOT submit
+        # the job and then immediately return a scheduled job id.
+        run-in-background = true
+
+        # `temporary-directory` creates the temporary directory for commands.
+        #
+        # If this value is not set explicitly, the default value creates a unique temporary directory, equivalent to:
+        # temporary-directory = "$(mktemp -d \"$PWD\"/tmp.XXXXXX)"
+        #
+        # The expression is run from the execution directory for the script. The expression must create the directory
+        # if it does not exist, and then return the full path to the directory.
+        #
+        # To create and return a non-random temporary directory, use something like:
+        # temporary-directory = "$(mkdir -p /tmp/mydir && echo /tmp/mydir)"
+
+        # `script-epilogue` configures a shell command to run after the execution of every command block.
+        #
+        # If this value is not set explicitly, the default value is `sync`, equivalent to:
+        # script-epilogue = "sync"
+        #
+        # To turn off the default `sync` behavior set this value to an empty string:
+        # script-epilogue = ""
+
+        # The list of possible runtime custom attributes.
+        runtime-attributes = """
+        String? docker
+        String? docker_user
+        """
+
+        # Submit string when there is no "docker" runtime attribute.
+        submit = "/usr/bin/env bash ${script}"
+
+        # Submit string when there is a "docker" runtime attribute.
+        submit-docker = """
+        docker run \
+          --rm -i \
+          ${"--user " + docker_user} \
+          --entrypoint ${job_shell} \
+          -v ${cwd}:${docker_cwd} \
+          ${docker} ${script}
+        """
+
+        # Root directory where Cromwell writes job results.  This directory must be
+        # visible and writeable by the Cromwell process as well as the jobs that Cromwell
+        # launches.
+        root = "cromwell-executions"
+
+        # Root directory where Cromwell writes job results in the container. This value
+        # can be used to specify where the execution folder is mounted in the container.
+        # it is used for the construction of the docker_cwd string in the submit-docker
+        # value above.
+        dockerRoot = "/cromwell-executions"
+
+        # File system configuration.
+        filesystems {
+
+          # For SFS backends, the "local" configuration specifies how files are handled.
+          local {
+
+            # Try to hard link (ln), then soft-link (ln -s), and if both fail, then copy the files.
+            localization: [
+              "hard-link", "soft-link", "copy"
+            ]
+
+            # Call caching strategies
+            caching {
+              # When copying a cached result, what type of file duplication should occur. Attempted in the order listed below:
+              duplication-strategy: [
+                "hard-link", "soft-link", "copy"
+              ]
+
+              # Possible values: file, path, path+modtime
+              # "file" will compute an md5 hash of the file content.
+              # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link",
+              # in order to allow for the original file path to be hashed.
+              # "path+modtime" will compute an md5 hash of the file path and the last modified time. The same conditions as for "path" apply here.
+              # Default: file
+              hashing-strategy: "file"
+
+              # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash.
+              # If false or the md5 does not exist, will proceed with the above-defined hashing strategy.
+              check-sibling-md5: false
+            }
+          }
+        }
+
+        # The defaults for runtime attributes if not provided.
+        default-runtime-attributes {
+          failOnStderr: false
+          continueOnReturnCode: 0
+        }
+      }
+    }