Merge batch

README.md

@@ -91,4 +91,4 @@ open-source by providing free hosting, and YourKit, LLC for generously providing
 free licenses for <a href="https://www.yourkit.com/java/profiler/">YourKit Java
 Profiler</a> for open-source development.
 
-<img src="https://www.yourkit.com/images/yklogo.png" align="right" />
+<img src="https://www.yourkit.com/images/yklogo.png" align="right" />

batch/.dockerignore

@@ -0,0 +1,5 @@
+batch.log
+logs
+*~
+*.pyc
+**/__pycache__

batch/.gitignore

@@ -0,0 +1,5 @@
+batch.log
+logs
+*~
+*.pyc
+**/__pycache__

batch/Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:3.8
+
+RUN apk update
+RUN apk add python3 py3-cffi py3-cryptography
+RUN pip3 install -U pip
+RUN pip install flask
+RUN pip install kubernetes
+RUN pip install cerberus
+
+COPY batch /batch
+
+EXPOSE 5000
+
+CMD ["python3", "/batch/server.py"]

batch/Dockerfile.test

@@ -0,0 +1,13 @@
+FROM alpine:3.8
+
+RUN apk update
+RUN apk add python3 # python3=3.6.4-r1
+RUN pip3 install -U pip
+RUN pip install flask
+RUN pip install kubernetes
+RUN pip install cerberus
+
+COPY batch /batch
+COPY test /test
+
+CMD ["python3", "-m", "unittest", "/test/test_batch.py"]

batch/Makefile

@@ -0,0 +1,40 @@
+.PHONY: hail-ci-build-image push-hail-ci-build-image
+
+hail-ci-build-image:
+	docker build -t batch-pr-builder -f Dockerfile.pr-builder .
+	echo "gcr.io/broad-ctsa/batch-pr-builder:`docker images -q --no-trunc batch-pr-builder | sed -e 's,[^:]*:,,'`" > ../hail-ci-build-image
+	docker tag batch-pr-builder `cat ../hail-ci-build-image`
+
+push-hail-ci-build-image: hail-ci-build-image
+	docker push `cat ../hail-ci-build-image`
+
+build: build-batch build-batch-test
+
+build-batch:
+	docker build -t batch .
+
+build-batch-test:
+	docker build -t batch-test -f Dockerfile.test .
+
+push: push-batch push-batch-test
+
+push-batch: IMAGE="gcr.io/broad-ctsa/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')"
+push-batch: build-batch
+	echo $(IMAGE) > batch-image
+	docker tag batch $(IMAGE)
+	docker push $(IMAGE)
+
+push-batch-test: IMAGE="gcr.io/broad-ctsa/batch-test:$(shell docker images -q --no-trunc batch-test | sed -e 's,[^:]*:,,')"
+push-batch-test: build-batch-test
+	echo $(IMAGE) > batch-test-image
+	docker tag batch $(IMAGE)
+	docker push $(IMAGE)
+
+run-docker:
+	docker run -e BATCH_USE_KUBE_CONFIG=1 -i -v $(HOME)/.kube:/root/.kube -p 5000:5000 -t batch
+
+run:
+	BATCH_USE_KUBE_CONFIG=1 python batch/server.py
+
+test-local:
+	POD_IP='127.0.0.1' BATCH_URL='http://127.0.0.1:5000' python -m unittest -v test/test_batch.py

batch/README.md

@@ -0,0 +1,196 @@
+Getting Started
+---
+
+Start a `minikube` k8s cluster and configure your `kubectl` to point at that k8s
+cluster:
+
+```
+minikube start
+```
+
+If you get a weird minikube error, try
+
+```
+minikube delete
+rm -rf ~/.minikube
+brew cask reinstall minikube # or equivalent on your OS
+minikube start
+```
+
+When you want to return to using a google k8s cluster, you can run this:
+
+```
+gcloud container clusters get-credentials CLUSTER_NAME
+```
+
+Set some environment variables so that docker images are placed in the
+`minikube` cluster's docker registry:
+
+```
+eval $(minikube docker-env)
+```
+
+Build the batch and test image
+
+```
+make build-batch build-test
+```
+
+edit the `deployment.yaml` so that the container named `batch` has
+`imagePullPolicy: Never`. This ensures that k8s does not go look for the image
+in the Google Container Registry and instead uses the local image cache (which
+you just updated when you ran `make build-batch build-test`).
+
+Give way too many privileges to the default service account so that `batch` can
+start new pods:
+
+```
+kubectl create clusterrolebinding \
+  cluster-admin-default \
+  --clusterrole cluster-admin \
+  --serviceaccount=default:default
+```
+
+Create a batch service:
+
+```
+kubectl create -f deployment.yaml
+```
+
+If you ever need to shutdown the service, execute:
+
+```
+kubectl delete -f deployment.yaml
+```
+
+Look for the newly created batch pod:
+
+```
+kubectl get pods
+```
+
+And create a port forward from the k8s cluster to your local machine (this works
+for clusters in GKE too):
+
+```
+kubectl port-forward POD_NAME 5000:5000
+```
+
+The former port is the local one and the latter port is the remote one (i.e. in
+the k8s pod). Now you can load the conda environment for testing and run the
+tests against this deployment:
+
+```
+conda env create -f environment.yaml
+conda activate hail-batch
+make test-local
+```
+
+
+
+---
+
+Kubernetes [Python client](https://github.com/kubernetes-client/python/blob/master/kubernetes/README.md)
+ - [V1Pod](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1Pod.md)
+ - [create_namespaced_pod](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/CoreV1Api.md#create_namespaced_pod)
+ - [delete_namespaced_pod](https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/CoreV1Api.md#delete_namespaced_pod)
+ - 
+
+To get kubectl credentials for a GKE cluster:
+
+```
+$ gcloud container clusters get-credentials <cluster>
+```
+
+To authorize docker to push to GCR:
+
+```
+$ gcloud auth configure-docker
+```
+
+To run batch locally, using the local kube credentials:
+
+```
+$ docker run -i -v $HOME/.kube:/root/.kube -p 5000:5000 -t batch
+```
+
+On OSX, the port will be accessible on the docker-machine:
+
+```
+$(docker-machine ip default):5000
+```
+
+Get a shell in a running pod:
+
+```
+$ kubectl exec -it <pod> -- /bin/sh
+```
+
+Hit a Flask REST endpoint with Curl:
+
+```
+$ curl -X POST -H "Content-Type: application/json" -d <data> <url>
+$ curl -X POST -H "Content-Type: application/json" -d '{"name": "batchtest", "image": "gcr.io/broad-ctsa/true"}' batch/jobs/create
+```
+
+Give default:default serviceaccount cluster-admin privileges:
+
+```
+$ kubectl create clusterrolebinding cluster-admin-default --clusterrole cluster-admin --serviceaccount=default:default
+```
+
+Run an image in a new pod:
+
+```
+$ kubectl run <name> --restart=Never --image <image> -- <cmd>
+```
+
+For example, run a shell in an new pod:
+
+```
+$ kubectl run -i --tty apline --image=alpine --restart=Never -- sh
+```
+
+Forward from a local port to a port on pod:
+
+```
+$ kubectl port-forward jupyter-deployment-5f54cff675-msr85 8888:8888 # <local port>:<remote port>
+```
+
+Run container with a given hostname:
+
+$ docker run -d --rm --name spark-m -h spark-m -p 8080:8080 -p 7077:7077 spark-m
+
+List all containers, included stopped containers:
+
+$ docker ps -a
+
+Remove all stopped containers:
+
+$ docker ps -aq --no-trunc -f status=exited | xargs docker rm
+
+Run a docker container linked to another:
+
+$ docker run -d --rm --cpus 0.5 --name spark-w-0 --link spark-m spark-w -c 1 -m 2g
+
+Get IP of container:
+
+$ docker inspect <container-id> | grep IPAddress
+
+---
+
+The following will set some environment variables so that future invocations of
+`docker build` will make images available to the minikube cluster. This allows
+you to test images without pushing them to a remote container registry.
+
+```
+eval $(minikube docker-env)
+make build-batch build-test
+```
+
+NB: you must also set the `imagePullPolicy` of any `container` you `kubectl
+create` to `Never` if you're using the `:latest` image tag (which is implicitly
+used if no tag is specified on the image name). Otherwise, k8s will always try
+to check if there is a newer version of the image. Even if `imagePullPolicy`
+is set to `NotIfPresent`, k8s will still check for a newer image if you use the
+`:latest` tag.

batch/batch/__init__.py

@@ -0,0 +1,2 @@
+import batch.client
+import batch.api

batch/batch/api.py

@@ -0,0 +1,64 @@
+import json
+import time
+import random
+import requests
+
+def create_job(url, spec, attributes, batch_id, callback):
+    d = {'spec': spec}
+    if attributes:
+        d['attributes'] = attributes
+    if batch_id:
+        d['batch_id'] = batch_id
+    if callback:
+        d['callback'] = callback
+
+    r = requests.post(url + '/jobs/create', json = d)
+    r.raise_for_status()
+    return r.json()
+
+def list_jobs(url):
+    r = requests.get(url + '/jobs')
+    r.raise_for_status()
+    return r.json()
+
+def get_job(url, job_id):
+    r = requests.get(url + '/jobs/{}'.format(job_id))
+    r.raise_for_status()
+    return r.json()
+
+def get_job_log(url, job_id):
+    r = requests.get(url + '/jobs/{}/log'.format(job_id))
+    r.raise_for_status()
+    return r.text
+
+def delete_job(url, job_id):
+    r = requests.delete(url + '/jobs/{}/delete'.format(job_id))
+    r.raise_for_status()
+    return r.json()
+
+def cancel_job(url, job_id):
+    r = requests.post(url + '/jobs/{}/cancel'.format(job_id))
+    r.raise_for_status()
+    return r.json()
+
+def create_batch(url, attributes):
+    d = {}
+    if attributes:
+        d['attributes'] = attributes
+    r = requests.post(url + '/batches/create', json = d)
+    r.raise_for_status()
+    return r.json()
+
+def get_batch(url, batch_id):
+    r = requests.get(url + '/batches/{}'.format(batch_id))
+    r.raise_for_status()
+    return r.json()
+
+def delete_batch(url, batch_id):
+    r = requests.delete(url + '/batches/{}'.format(batch_id))
+    r.raise_for_status()
+    return r.json()
+
+def refresh_k8s_state(url):
+    r = requests.post(url + '/refresh_k8s_state')
+    r.raise_for_status()