Upstream merge

Makefile

@@ -75,13 +75,13 @@ install-dev-requirements:
 hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python/hailtop
 
-hail/python/pinned-requirements.txt: hail/python/requirements.txt hail/python/hailtop/pinned-requirements.txt
+hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt hail/python/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python
 
-hail/python/dev/pinned-requirements.txt: hail/python/dev/requirements.txt hail/python/pinned-requirements.txt
+hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
 	./generate-linux-pip-lockfile.sh hail/python/dev
 
-gear/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
+gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
 	./generate-linux-pip-lockfile.sh gear
 
 web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt
@@ -165,14 +165,10 @@ hail-buildkit-image: ci/buildkit/Dockerfile
 	./docker-build.sh ci buildkit/Dockerfile.out $(HAIL_BUILDKIT_IMAGE)
 	echo $(HAIL_BUILDKIT_IMAGE) > $@
 
-batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar:
-	mkdir -p batch/jars
-	cd batch/jars && curl -LO https://github.com/kohlschutter/junixsocket/releases/download/junixsocket-parent-2.3.3/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
+batch/jvm-entryway/build/libs/jvm-entryway.jar: $(shell git ls-files batch/jvm-entryway)
+	cd batch/jvm-entryway && ./gradlew shadowJar
 
-batch/src/main/java/is/hail/JVMEntryway.class: batch/src/main/java/is/hail/JVMEntryway.java batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar
-	javac -cp batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar $<
-
-batch-worker-image: batch/src/main/java/is/hail/JVMEntryway.class $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch)
+batch-worker-image: batch/jvm-entryway/build/libs/jvm-entryway.jar $(SERVICES_IMAGE_DEPS) $(shell git ls-files batch)
 	$(eval BATCH_WORKER_IMAGE := $(DOCKER_PREFIX)/batch-worker:$(TOKEN))
 	python3 ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"'$$(cat hail-ubuntu-image)'"},"global":{"cloud":"$(CLOUD)"}}' batch/Dockerfile.worker batch/Dockerfile.worker.out
 	./docker-build.sh . batch/Dockerfile.worker.out $(BATCH_WORKER_IMAGE)

auth/pinned-requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.7
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile --output-file=hail/auth/pinned-requirements.txt hail/auth/requirements.txt
 #
@@ -22,7 +22,7 @@ charset-normalizer==3.1.0
     #   -c hail/auth/../hail/python/pinned-requirements.txt
     #   -c hail/auth/../web_common/pinned-requirements.txt
     #   requests
-google-auth==2.17.2
+google-auth==2.17.3
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt
@@ -38,13 +38,13 @@ idna==3.4
     #   requests
 oauthlib==3.2.2
     # via requests-oauthlib
-pyasn1==0.4.8
+pyasn1==0.5.0
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.2.8
+pyasn1-modules==0.3.0
     # via
     #   -c hail/auth/../gear/pinned-requirements.txt
     #   -c hail/auth/../hail/python/pinned-requirements.txt

batch/Dockerfile.worker

@@ -8,7 +8,7 @@ RUN hail-apt-get-install \
     xfsprogs \
     libyajl-dev # crun runtime dependency
 
-RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
 
 {% if global.cloud == "gcp" %}
 RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \
@@ -39,7 +39,7 @@ RUN hail-pip-install \
     -r batch-requirements.txt \
     pyspark==3.3.0
 
-ENV SPARK_HOME /usr/local/lib/python3.7/dist-packages/pyspark
+ENV SPARK_HOME /usr/local/lib/python3.8/dist-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"
 ENV PYSPARK_PYTHON python3
 
@@ -71,6 +71,5 @@ COPY batch/batch /batch/batch/
 
 RUN hail-pip-install /hailtop /gear /batch
 
-COPY batch/jars/junixsocket-selftest-2.3.3-jar-with-dependencies.jar /jvm-entryway/
-COPY batch/src/main/java/is /jvm-entryway/is
+COPY batch/jvm-entryway/build/libs/jvm-entryway.jar /jvm-entryway/
 COPY letsencrypt/subdomains.txt /

batch/batch/cloud/azure/worker/credentials.py

@@ -6,30 +6,14 @@
 
 
 class AzureUserCredentials(CloudUserCredentials):
-    def __init__(self, data: Dict[str, bytes]):
+    def __init__(self, data: Dict[str, str]):
         self._data = data
         self._credentials = json.loads(base64.b64decode(data['key.json']).decode())
 
-    @property
-    def secret_name(self) -> str:
-        return 'azure-credentials'
-
-    @property
-    def secret_data(self) -> Dict[str, bytes]:
-        return self._data
-
-    @property
-    def file_name(self) -> str:
-        return 'key.json'
-
     @property
     def cloud_env_name(self) -> str:
         return 'AZURE_APPLICATION_CREDENTIALS'
 
-    @property
-    def hail_env_name(self) -> str:
-        return 'HAIL_AZURE_CREDENTIAL_FILE'
-
     @property
     def username(self):
         return self._credentials['appId']
@@ -40,7 +24,7 @@ def password(self):
 
     @property
     def mount_path(self):
-        return f'/{self.secret_name}/{self.file_name}'
+        return '/azure-credentials/key.json'
 
     def cloudfuse_credentials(self, fuse_config: dict) -> str:
         # https://github.com/Azure/azure-storage-fuse

batch/batch/cloud/azure/worker/worker_api.py

@@ -42,7 +42,7 @@ def get_compute_client(self) -> aioazure.AzureComputeClient:
         azure_config = get_azure_config()
         return aioazure.AzureComputeClient(azure_config.subscription_id, azure_config.resource_group)
 
-    def user_credentials(self, credentials: Dict[str, bytes]) -> AzureUserCredentials:
+    def user_credentials(self, credentials: Dict[str, str]) -> AzureUserCredentials:
         return AzureUserCredentials(credentials)
 
     async def worker_access_token(self, session: httpx.ClientSession) -> Dict[str, str]:

batch/batch/cloud/gcp/worker/credentials.py

@@ -5,40 +5,24 @@
 
 
 class GCPUserCredentials(CloudUserCredentials):
-    def __init__(self, data: Dict[str, bytes]):
+    def __init__(self, data: Dict[str, str]):
         self._data = data
 
-    @property
-    def secret_name(self) -> str:
-        return 'gsa-key'
-
-    @property
-    def secret_data(self) -> Dict[str, bytes]:
-        return self._data
-
-    @property
-    def file_name(self) -> str:
-        return 'key.json'
-
     @property
     def cloud_env_name(self) -> str:
         return 'GOOGLE_APPLICATION_CREDENTIALS'
 
-    @property
-    def hail_env_name(self) -> str:
-        return 'HAIL_GSA_KEY_FILE'
-
     @property
     def username(self):
         return '_json_key'
 
     @property
     def password(self) -> str:
-        return base64.b64decode(self.secret_data['key.json']).decode()
+        return base64.b64decode(self._data['key.json']).decode()
 
     @property
     def mount_path(self):
-        return f'/{self.secret_name}/{self.file_name}'
+        return '/gsa-key/key.json'
 
     def cloudfuse_credentials(self, fuse_config: dict) -> str:  # pylint: disable=unused-argument
         return self.password

batch/batch/cloud/gcp/worker/worker_api.py

@@ -47,7 +47,7 @@ def get_cloud_async_fs(self) -> aiogoogle.GoogleStorageAsyncFS:
     def get_compute_client(self) -> aiogoogle.GoogleComputeClient:
         return self._compute_client
 
-    def user_credentials(self, credentials: Dict[str, bytes]) -> GCPUserCredentials:
+    def user_credentials(self, credentials: Dict[str, str]) -> GCPUserCredentials:
         return GCPUserCredentials(credentials)
 
     async def worker_access_token(self, session: httpx.ClientSession) -> Dict[str, str]:

batch/batch/front_end/front_end.py

@@ -1029,6 +1029,7 @@ async def _create_jobs(userdata: dict, job_specs: dict, batch_id: int, update_id
     db: Database = app['db']
     file_store: FileStore = app['file_store']
     user = userdata['username']
+    is_developer = userdata['is_developer']
 
     # restrict to what's necessary; in particular, drop the session
     # which is sensitive
@@ -1271,6 +1272,10 @@ async def _create_jobs(userdata: dict, job_specs: dict, batch_id: int, update_id
         if cloud == 'azure' and all(envvar['name'] != 'AZURE_APPLICATION_CREDENTIALS' for envvar in spec['env']):
             spec['env'].append({'name': 'AZURE_APPLICATION_CREDENTIALS', 'value': '/gsa-key/key.json'})
 
+        cloudfuse = spec.get('gcsfuse') or spec.get('cloudfuse')
+        if not is_developer and user not in ('ci', 'test', 'test-dev') and cloudfuse is not None and len(cloudfuse) > 0:
+            raise web.HTTPBadRequest(reason='cloudfuse requests are temporarily not supported.')
+
         if spec.get('mount_tokens', False):
             secrets.append(
                 {

batch/batch/globals.py

@@ -30,7 +30,7 @@
 
 BATCH_FORMAT_VERSION = 7
 STATUS_FORMAT_VERSION = 5
-INSTANCE_VERSION = 23
+INSTANCE_VERSION = 24
 
 MAX_PERSISTENT_SSD_SIZE_GIB = 64 * 1024
 RESERVED_STORAGE_GB_PER_CORE = 5

batch/batch/worker/credentials.py

@@ -1,39 +1,26 @@
 import abc
-from typing import Dict
 
 
 class CloudUserCredentials(abc.ABC):
     @property
-    def secret_name(self) -> str:
-        raise NotImplementedError
-
-    @property
-    def secret_data(self) -> Dict[str, bytes]:
-        raise NotImplementedError
-
-    @property
-    def file_name(self) -> str:
-        raise NotImplementedError
-
-    @property
+    @abc.abstractmethod
     def cloud_env_name(self) -> str:
         raise NotImplementedError
 
     @property
-    def hail_env_name(self) -> str:
-        raise NotImplementedError
-
-    @property
+    @abc.abstractmethod
     def username(self) -> str:
         raise NotImplementedError
 
     @property
+    @abc.abstractmethod
     def password(self) -> str:
         raise NotImplementedError
 
     @property
+    @abc.abstractmethod
     def mount_path(self):
-        return f'/{self.secret_name}/{self.file_name}'
+        raise NotImplementedError
 
     @abc.abstractmethod
     def cloudfuse_credentials(self, fuse_config: dict) -> str: