Upstream v0.2.113

.gitignore

@@ -35,12 +35,13 @@ hail/upload-qob-jar
 hail/upload-qob-test-resources
 **/hail_version
 web_common/web_common/static/css/
-website/docs.tar.gz
+docs.tar.gz
 website/website/static/css/
 *.dylib  # macOS dynamic libraries
 */hail.jar
 infra/.terraform.lock.hcl
 .dist/
 .cache/
 wheel-container.tar
+*-image
 _/

Makefile

@@ -1,5 +1,27 @@
 .DEFAULT_GOAL := default
 
+include config.mk
+
+SERVICES := auth batch ci memory notebook monitoring website
+SERVICES_IMAGES := $(patsubst %, %-image, $(SERVICES))
+SERVICES_MODULES := $(SERVICES) gear web_common
+CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
+
+HAILTOP_VERSION := hail/python/hailtop/hail_version
+SERVICES_IMAGE_DEPS = hail-ubuntu-image $(HAILTOP_VERSION) $(shell git ls-files hail/python/hailtop gear web_common)
+
+EMPTY :=
+SPACE := $(EMPTY) $(EMPTY)
+EXTRA_PYTHONPATH := hail/python:$(subst $(SPACE),:,$(SERVICES_MODULES))
+
+PYTHONPATH ?= ""
+ifeq ($(PYTHONPATH), "")
+PYTHONPATH := $(EXTRA_PYTHONPATH)
+else
+PYTHONPATH := $(PYTHONPATH):$(EXTRA_PYTHONPATH)
+endif
+PYTHON := PYTHONPATH="$(PYTHONPATH)" python3
+
 default:
 	@echo Do not use this makefile to build hail, for information on how to \
 	     build hail see: https://hail.is/docs/0.2/
@@ -13,46 +35,145 @@ check-hail:
 	$(MAKE) -C hail/python check
 
 .PHONY: check-services
-check-services: check-auth check-batch check-ci check-gear check-memory \
-  check-notebook check-monitoring check-web-common check-website
+check-services: $(CHECK_SERVICES_MODULES)
+
+.PHONY: check-%
+$(CHECK_SERVICES_MODULES): check-%:
+	$(PYTHON) -m flake8  --config setup.cfg $*
+	$(PYTHON) -m pylint --rcfile pylintrc --recursive=y $* --score=n
+	$(PYTHON) -m mypy --config-file setup.cfg $*
+	$(PYTHON) -m isort $* --check-only --diff
+	$(PYTHON) -m black $* --line-length=120 --skip-string-normalization --check --diff
+	curlylint $*
+	cd $* && bash ../check-sql.sh
+
+.PHONY: check-pip-requirements
+check-pip-requirements:
+	./check_pip_requirements.sh \
+		hail/python/hailtop \
+		hail/python \
+		hail/python/dev \
+		gear \
+		web_common \
+		auth \
+		batch \
+		ci \
+		memory
+
+.PHONY: install-dev-requirements
+install-dev-requirements:
+	python3 -m pip install \
+		-r hail/python/pinned-requirements.txt \
+		-r hail/python/dev/pinned-requirements.txt \
+		-r gear/pinned-requirements.txt \
+		-r web_common/pinned-requirements.txt \
+		-r auth/pinned-requirements.txt \
+		-r batch/pinned-requirements.txt \
+		-r ci/pinned-requirements.txt \
+		-r memory/pinned-requirements.txt \
+
+hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
+	./generate-linux-pip-lockfile.sh hail/python/hailtop
+
+hail/python/pinned-requirements.txt: hail/python/requirements.txt hail/python/hailtop/pinned-requirements.txt
+	./generate-linux-pip-lockfile.sh hail/python
+
+hail/python/dev/pinned-requirements.txt: hail/python/dev/requirements.txt hail/python/pinned-requirements.txt
+	./generate-linux-pip-lockfile.sh hail/python/dev
+
+gear/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
+	./generate-linux-pip-lockfile.sh gear
+
+web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt
+	./generate-linux-pip-lockfile.sh web_common
+
+auth/pinned-requirements.txt: web_common/pinned-requirements.txt auth/requirements.txt
+	./generate-linux-pip-lockfile.sh auth
+
+batch/pinned-requirements.txt: web_common/pinned-requirements.txt batch/requirements.txt
+	./generate-linux-pip-lockfile.sh batch
+
+ci/pinned-requirements.txt: web_common/pinned-requirements.txt ci/requirements.txt
+	./generate-linux-pip-lockfile.sh ci
+
+memory/pinned-requirements.txt: gear/pinned-requirements.txt memory/requirements.txt
+	./generate-linux-pip-lockfile.sh memory
+
+.PHONY: generate-pip-lockfiles
+generate-pip-lockfiles: hail/python/hailtop/pinned-requirements.txt
+generate-pip-lockfiles: hail/python/pinned-requirements.txt
+generate-pip-lockfiles: hail/python/dev/pinned-requirements.txt
+generate-pip-lockfiles: gear/pinned-requirements.txt
+generate-pip-lockfiles: web_common/pinned-requirements.txt
+generate-pip-lockfiles: auth/pinned-requirements.txt
+generate-pip-lockfiles: batch/pinned-requirements.txt
+generate-pip-lockfiles: ci/pinned-requirements.txt
+generate-pip-lockfiles: memory/pinned-requirements.txt
+
+$(HAILTOP_VERSION):
+	$(MAKE) -C hail python/hailtop/hail_version
+
+hail-ubuntu-image: $(shell git ls-files docker/hail-ubuntu)
+	$(eval HAIL_UBUNTU_IMAGE := $(DOCKER_PREFIX)/hail-ubuntu:$(TOKEN))
+	python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' docker/hail-ubuntu/Dockerfile docker/hail-ubuntu/Dockerfile.out
+	./docker-build.sh docker/hail-ubuntu Dockerfile.out $(HAIL_UBUNTU_IMAGE)
+	echo $(HAIL_UBUNTU_IMAGE) > $@
 
-.PHONY: check-auth
-check-auth:
-	$(MAKE) -C auth check
+base-image: hail-ubuntu-image docker/Dockerfile.base
+	$(eval BASE_IMAGE := $(DOCKER_PREFIX)/base:$(TOKEN))
+	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/Dockerfile.base docker/Dockerfile.base.out
+	./docker-build.sh . docker/Dockerfile.base.out $(BASE_IMAGE)
+	echo $(BASE_IMAGE) > $@
 
-.PHONY: check-batch
-check-batch:
-	$(MAKE) -C batch check
+private-repo-hailgenetics-hail-image: hail-ubuntu-image docker/hailgenetics/hail/Dockerfile $(shell git ls-files hail/src/main hail/python)
+	$(eval PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE := $(DOCKER_PREFIX)/hailgenetics/hail:$(TOKEN))
+	$(MAKE) -C hail wheel
+	tar -cvf wheel-container.tar \
+		-C hail/build/deploy/dist \
+		hail-$$(cat hail/python/hail/hail_pip_version)-py3-none-any.whl
+	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' docker/hailgenetics/hail/Dockerfile docker/hailgenetics/hail/Dockerfile.out
+	./docker-build.sh . docker/hailgenetics/hail/Dockerfile.out $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE)
+	rm wheel-container.tar
+	echo $(PRIVATE_REPO_HAILGENETICS_HAIL_IMAGE) > $@
 
-.PHONY: check-ci
-check-ci:
-	$(MAKE) -C ci check
+.PHONY: docs
+docs:
+	$(MAKE) -C hail hail-docs-no-test batch-docs
+	gcloud storage cp gs://hail-common/builds/0.1/docs/hail-0.1-docs-5a6778710097.tar.gz .
+	mkdir -p hail/build/www/docs/0.1
+	tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2
+	rm hail-0.1-docs-5a6778710097.tar.gz
+	tar czf docs.tar.gz -C hail/build/www .
 
-.PHONY: check-gear
-check-gear:
-	$(MAKE) -C gear check
+website-image: docs
 
-.PHONY: check-memory
-check-memory:
-	$(MAKE) -C memory check
+$(SERVICES_IMAGES): %-image: $(SERVICES_IMAGE_DEPS) $(shell git ls-files $$*)
+	$(eval IMAGE := $(DOCKER_PREFIX)/$*:$(TOKEN))
+	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"}}' $*/Dockerfile $*/Dockerfile.out
+	./docker-build.sh . $*/Dockerfile.out $(IMAGE)
+	echo $(IMAGE) > $@
 
-.PHONY: check-notebook
-check-notebook:
-	$(MAKE) -C notebook check
+ci-utils-image: base-image $(SERVICES_IMAGE_DEPS) $(shell git ls-files ci)
+	$(eval CI_UTILS_IMAGE := $(DOCKER_PREFIX)/ci-utils:$(TOKEN))
+	python3 ci/jinja2_render.py '{"base_image":{"image":"'$$(cat base-image)'"}}' ci/Dockerfile.ci-utils ci/Dockerfile.ci-utils.out
+	./docker-build.sh . ci/Dockerfile.ci-utils.out $(CI_UTILS_IMAGE)
+	echo $(CI_UTILS_IMAGE) > $@
 
-.PHONY: check-monitoring
-	$(MAKE) -C monitoring check
+hail-buildkit-image: ci/buildkit/Dockerfile
+	$(eval HAIL_BUILDKIT_IMAGE := $(DOCKER_PREFIX)/hail-buildkit:$(TOKEN))
+	python3 ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' ci/buildkit/Dockerfile ci/buildkit/Dockerfile.out
+	./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE)
+	echo $(HAIL_BUILDKIT_IMAGE) > $@
 
-.PHONY: check-web-common
-check-web-common:
-	$(MAKE) -C web_common check
+batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:
+	mkdir -p batch/jars
+	cd batch/jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
 
-.PHONY: check-website
-check-website:
-	$(MAKE) -C website check
+batch/src/main/java/is/hail/JVMEntryway.class: batch/src/main/java/is/hail/JVMEntryway.java batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
+	javac -cp batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $<
 
-.PHONY: check-pip-dependencies
-check-pip-dependencies:
-	./check_pip_requirements.sh hail/python/requirements.txt hail/python/pinned-requirements.txt
-	./check_pip_requirements.sh hail/python/dev/requirements.txt hail/python/dev/pinned-requirements.txt
-	./check_pip_requirements.sh docker/requirements.txt docker/linux-pinned-requirements.txt
+batch-worker-image: batch/src/main/java/is/hail/JVMEntryway.class $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch)
+	$(eval BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN))
+	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"},"global":{"cloud":"$(CLOUD)"}}' batch/Dockerfile.worker batch/Dockerfile.worker.out
+	./docker-build.sh . batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE)
+	echo $(BATCH_WORKER_IMAGE) > $@

admin-pod/Dockerfile

@@ -0,0 +1,8 @@
+FROM {{ hail_ubuntu_image.image }}
+
+# https://bugs.mysql.com/bug.php?id=105288&thanks=sub
+RUN hail-apt-get-install xz-utils libncurses5 && \
+    curl --remote-name https://downloads.mysql.com/archives/get/p/23/file/mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild.tar.xz && \
+    mkdir -p /opt && \
+    tar -vx -C /opt -f mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild.tar.xz && \
+    ln -s /opt/mysql-8.0.26-linux-glibc2.17-x86_64-minimal-rebuild/bin/* /usr/bin/

admin-pod/Makefile

@@ -1,8 +1,19 @@
 include ../config.mk
 
+ADMIN_POD_IMAGE := $(DOCKER_PREFIX)/admin-pod:$(TOKEN)
+
+.PHONY: hail-ubuntu
+hail-ubuntu:
+	$(MAKE) -C ../docker hail-ubuntu
+
+.PHONY: build
+build: hail-ubuntu
+	python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat ../docker/hail-ubuntu-image-ref)'"}}' Dockerfile Dockerfile.out
+	../docker-build.sh . Dockerfile.out $(ADMIN_POD_IMAGE)
+
 .PHONY: deploy
 deploy:
 	! [ -z $(NAMESPACE) ]  # call this like: make deploy NAMESPACE=default
 	$(MAKE) -C ../docker service-base
-	python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"service_base_image":{"image":"'$$(cat ../docker/service-base-image-ref)'"}}' admin-pod.yaml admin-pod.yaml.out
+	python3 ../ci/jinja2_render.py '{"deploy":$(DEPLOY),"admin_pod_image":{"image":"$(ADMIN_POD_IMAGE)"}}' admin-pod.yaml admin-pod.yaml.out
 	kubectl -n $(NAMESPACE) apply -f admin-pod.yaml.out

admin-pod/admin-pod.yaml

@@ -31,7 +31,7 @@ spec:
               ln -s /ssl-config/ssl-config.curlrc $HOME/.curlrc
               ln -s /sql-config/sql-config.cnf $HOME/.my.cnf
               while true; do sleep 1000; done
-         image: {{ service_base_image.image }}
+         image: {{ admin_pod_image.image }}
          resources:
            requests:
              cpu: "1m"

auth/Dockerfile

@@ -1,7 +1,29 @@
-FROM {{ service_base_image.image }}
+FROM {{ hail_ubuntu_image.image }}
+
+COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
+COPY gear/pinned-requirements.txt gear-requirements.txt
+COPY web_common/pinned-requirements.txt web_common-requirements.txt
+COPY auth/pinned-requirements.txt auth-requirements.txt
+RUN hail-pip-install \
+      -r hailtop-requirements.txt \
+      -r gear-requirements.txt \
+      -r web_common-requirements.txt \
+      -r auth-requirements.txt
+
+COPY hail/python/setup-hailtop.py /hailtop/setup.py
+COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
+COPY hail/python/hailtop /hailtop/hailtop/
+
+COPY gear/setup.py /gear/setup.py
+COPY gear/gear /gear/gear/
+
+COPY web_common/setup.py web_common/MANIFEST.in /web_common/
+COPY web_common/web_common /web_common/web_common/
 
 COPY auth/setup.py auth/MANIFEST.in /auth/
 COPY auth/auth /auth/auth/
-RUN hail-pip-install /auth && rm -rf /auth
+
+RUN hail-pip-install /hailtop /gear /web_common /auth && \
+    rm -rf /hailtop /gear /web_common /auth
 
 EXPOSE 5000

auth/Makefile

@@ -1,31 +1,14 @@
 include ../config.mk
 
-AUTH_IMAGE := $(DOCKER_PREFIX)/auth:$(TOKEN)
-
-EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
-PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
-BLACK := $(PYTHON) -m black . --line-length=120 --skip-string-normalization
-
-.PHONY: check
-check:
-	$(PYTHON) -m flake8 auth
-	$(PYTHON) -m pylint --rcfile ../pylintrc auth --score=n
-	$(PYTHON) -m isort . --check-only --diff
-	$(BLACK) --check --diff
-	curlylint .
-	bash ../check-sql.sh
-
 .PHONY: build
 build:
-	$(MAKE) -C ../docker service-base
-	python3 ../ci/jinja2_render.py '{"service_base_image":{"image":"'$$(cat ../docker/service-base-image-ref)'"}}' Dockerfile Dockerfile.out
-	../docker-build.sh .. auth/Dockerfile.out $(AUTH_IMAGE)
+	$(MAKE) -C .. auth-image
 
 .PHONY: deploy
 deploy: build
 	! [ -z $(NAMESPACE) ]  # call this like: make deploy NAMESPACE=default
 	kubectl -n $(NAMESPACE) apply -f auth-driver-service-account.yaml
-	python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"},"scope":"$(SCOPE)","auth_image":{"image":"$(AUTH_IMAGE)"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
+	python3 ../ci/jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"scope":"$(SCOPE)","default_ns":{"name":"$(NAMESPACE)"},"auth_image":{"image":"'$$(cat ../auth-image)'"},"auth_database":{"user_secret_name":"sql-auth-user-config"}}' deployment.yaml deployment.yaml.out
 	kubectl -n $(NAMESPACE) apply -f deployment.yaml.out
 
 .PHONY:

auth/auth/driver/driver.py

@@ -5,7 +5,7 @@
 import os
 import random
 import secrets
-from typing import Optional
+from typing import Any, Awaitable, Callable, Dict, List, Optional
 
 import aiohttp
 import kubernetes_asyncio.client
@@ -390,9 +390,9 @@ async def _delete(self, user, billing_project):
             if e.status == 403 and 'Unknown Hail Batch billing project' in e.body:
                 return
             raise
-        else:
-            if bp['status'] == 'closed':
-                await self.batch_client.reopen_billing_project(billing_project)
+
+        if bp['status'] == 'closed':
+            await self.batch_client.reopen_billing_project(billing_project)
 
         try:
             await self.batch_client.remove_user(user, billing_project)
@@ -521,7 +521,7 @@ async def _create_user(app, user, skip_trial_bp, cleanup):
 
 
 async def create_user(app, user, skip_trial_bp=False):
-    cleanup = []
+    cleanup: List[Callable[[], Awaitable[None]]] = []
     try:
         await _create_user(app, user, skip_trial_bp, cleanup)
     except Exception:
@@ -610,7 +610,7 @@ async def update_users(app):
 
 
 async def async_main():
-    app = {}
+    app: Dict[str, Any] = {}
 
     user_creation_loop = None
     try:
@@ -629,7 +629,7 @@ async def async_main():
 
         app['identity_client'] = get_identity_client()
 
-        app['batch_client'] = await bc.aioclient.BatchClient.create(None)
+        app['batch_client'] = await bc.aioclient.BatchClient.create('')
 
         users_changed_event = asyncio.Event()
         app['users_changed_event'] = users_changed_event

auth/auth/flow.py

@@ -92,17 +92,17 @@ def initiate_flow(self, redirect_uri: str) -> dict:
         }
 
     def receive_callback(self, request: aiohttp.web.Request, flow_dict: dict) -> FlowResult:
-        query_dict = urllib.parse.parse_qs(request.query_string)
-        query_dict = {k: v[0] for k, v in query_dict.items()}
+        query_key_to_list_of_values = urllib.parse.parse_qs(request.query_string)
+        query_dict = {k: v[0] for k, v in query_key_to_list_of_values.items()}
 
         token = self._client.acquire_token_by_auth_code_flow(flow_dict['flow'], query_dict)
 
         if 'error' in token:
-            raise Exception(f'{token}')
+            raise ValueError(token)
 
         tid = token['id_token_claims']['tid']
         if tid != self._tenant_id:
-            raise Exception('invalid tenant id')
+            raise ValueError('invalid tenant id')
 
         return FlowResult(token['id_token_claims']['oid'], token['id_token_claims']['preferred_username'], token)