Merge upstream HEAD(bea04d9, 2024-05-21) [release] 0.2.130 (#14454)

.git-blame-ignore-revs

@@ -8,3 +8,11 @@
 4fccbe2d18c6d2f4059036d61489467c780bbc0e
 # Delete `FastIndexedSeq`
 fcc9ffab3cd68c4cfb26a1553d65118797c59d6e
+# blackens hail directory
+da2790242a40ec425a53a02707d261c893b264f7
+# scalafmt
+422edf6386616711ca70f87c455f76781ac925d4
+# replaces black formatting with ruff
+fa2ef0f2c76654d0c037ff6db60ccb8842fb8539
+# ruff lint python imports
+01a6a6a107faf204d4f5c20f8ae510d2c35518e9

.gitignore

@@ -50,3 +50,4 @@ hail/python/hail/backend/extra_classpath
 hail/python/hail/backend/hail.jar
 hail/install-editable
 _/
+.helix

.pre-commit-config.yaml

@@ -9,24 +9,26 @@ repos:
         language: system
         types: [python]
         require_serial: true
+      - id: ruff-format
+        name: ruff-format
+        entry: ruff format
+        language: system
+        types: [python]
+        require_serial: true
       - id: pyright
         name: pyright
         entry: pyright
         language: system
         types: [python]
         require_serial: true
+        exclude: hail/python/(hail|test)
         stages:
           - pre-push
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v2.3.0
     hooks:
       - id: end-of-file-fixer
       - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 22.3.0
-    hooks:
-      - id: black
-        language_version: python3
   - repo: https://github.com/thibaudcolas/curlylint
     rev: v0.13.1
     hooks:

Makefile

@@ -11,7 +11,7 @@ CHECK_SERVICES_MODULES := $(patsubst %, check-%, $(SERVICES_MODULES))
 SPECIAL_IMAGES := hail-ubuntu batch-worker letsencrypt
 
 HAILGENETICS_IMAGES = $(foreach img,hail vep-grch37-85 vep-grch38-95,hailgenetics-$(img))
-CI_IMAGES = ci-utils ci-buildkit base hail-run
+CI_IMAGES = ci-utils hail-buildkit base hail-run
 PRIVATE_REGISTRY_IMAGES = $(patsubst %, pushed-private-%-image, $(SPECIAL_IMAGES) $(SERVICES_PLUS_ADMIN_POD) $(CI_IMAGES) $(HAILGENETICS_IMAGES))
 
 HAILTOP_VERSION := hail/python/hailtop/hail_version
@@ -39,17 +39,21 @@ check-all: check-hail check-services
 
 .PHONY: check-hail-fast
 check-hail-fast:
-	ruff check hail/python/hail
-	ruff check hail/python/hailtop
+	ruff check hail
+	ruff format hail --diff
 	$(PYTHON) -m pyright hail/python/hailtop
 
+	ruff check hail/python/test/hailtop/batch
+	$(PYTHON) -m pyright hail/python/test/hailtop/batch
+
 .PHONY: pylint-hailtop
 pylint-hailtop:
 	# pylint on hail is still a work in progress
 	$(PYTHON) -m pylint --rcfile pylintrc hail/python/hailtop --score=n
 
 .PHONY: check-hail
 check-hail: check-hail-fast pylint-hailtop
+	cd hail && sh millw __.checkFormat + __.fix --check
 
 .PHONY: check-services
 check-services: $(CHECK_SERVICES_MODULES)
@@ -61,8 +65,8 @@ pylint-%:
 .PHONY: check-%-fast
 check-%-fast:
 	ruff check $*
+	ruff format $* --diff
 	$(PYTHON) -m pyright $*
-	$(PYTHON) -m black $* --check --diff
 	curlylint $*
 	cd $* && bash ../check-sql.sh
 
@@ -82,7 +86,6 @@ check-pip-requirements:
 		hail/python/dev \
 		gear \
 		web_common \
-		auth \
 		batch \
 		ci
 
@@ -94,7 +97,6 @@ check-linux-pip-requirements:
 		hail/python/dev \
 		gear \
 		web_common \
-		auth \
 		batch \
 		ci
 
@@ -173,7 +175,8 @@ hail-0.1-docs-5a6778710097.tar.gz:
 	gcloud storage cp gs://hail-common/builds/0.1/docs/$@ .
 
 hail/build/www: hail-0.1-docs-5a6778710097.tar.gz $(shell git ls-files hail)
-	$(MAKE) -C hail hail-docs-no-test batch-docs
+	@echo !!! This target does not render the notebooks because it takes a long time !!!
+	$(MAKE) -C hail hail-docs-do-not-render-notebooks batch-docs
 	mkdir -p hail/build/www/docs/0.1
 	tar -xvf hail-0.1-docs-5a6778710097.tar.gz -C hail/build/www/docs/0.1 --strip-components 2
 	touch $@  # Copying into the dir does not necessarily touch it

auth/auth/auth.py

@@ -11,7 +11,6 @@
 import kubernetes_asyncio.client
 import kubernetes_asyncio.client.rest
 import kubernetes_asyncio.config
-import uvloop
 from aiohttp import web
 from prometheus_async.aio.web import server_stats  # type: ignore
 
@@ -33,11 +32,10 @@
 from gear.auth import AIOHTTPHandler, get_session_id
 from gear.cloud_config import get_global_config
 from gear.profiling import install_profiler_if_requested
-from hailtop import httpx
+from hailtop import httpx, uvloopx
 from hailtop.auth import AzureFlow, Flow, GoogleFlow, IdentityProvider
 from hailtop.config import get_deploy_config
 from hailtop.hail_logging import AccessLogger
-from hailtop.tls import internal_server_ssl_context
 from hailtop.utils import secret_alnum_string
 from web_common import render_template, set_message, setup_aiohttp_jinja2, setup_common_static_routes
 
@@ -56,8 +54,6 @@
 
 log = logging.getLogger('auth')
 
-uvloop.install()
-
 CLOUD = get_global_config()['cloud']
 DEFAULT_NAMESPACE = os.environ['HAIL_DEFAULT_NAMESPACE']
 
@@ -164,10 +160,10 @@ async def _insert(tx):
             return False
 
         await tx.execute_insertone(
-            '''
+            """
 INSERT INTO users (state, username, login_id, is_developer, is_service_account, hail_identity, hail_credentials_secret_name)
 VALUES (%s, %s, %s, %s, %s, %s, %s);
-''',
+""",
             (
                 'creating',
                 username,
@@ -482,9 +478,11 @@ async def rest_login(request: web.Request) -> web.Response:
     flow_data['callback_uri'] = callback_uri
 
     # keeping authorization_url and state for backwards compatibility
-    return json_response(
-        {'flow': flow_data, 'authorization_url': flow_data['authorization_url'], 'state': flow_data['state']}
-    )
+    return json_response({
+        'flow': flow_data,
+        'authorization_url': flow_data['authorization_url'],
+        'state': flow_data['state'],
+    })
 
 
 @routes.get('/api/v1alpha/oauth2-client')
@@ -511,10 +509,10 @@ async def post_create_role(request: web.Request, _) -> NoReturn:
     name = str(post['name'])
 
     role_id = await db.execute_insertone(
-        '''
+        """
 INSERT INTO `roles` (`name`)
 VALUES (%s);
-''',
+""",
         (name),
     )
 
@@ -564,10 +562,10 @@ async def rest_get_users(request: web.Request, userdata: UserData) -> web.Respon
         raise web.HTTPUnauthorized()
 
     db = request.app[AppKeys.DB]
-    _query = '''
+    _query = """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity
 FROM users;
-'''
+"""
     users = [x async for x in db.select_and_fetchall(_query)]
     return json_response(users)
 
@@ -579,10 +577,10 @@ async def rest_get_user(request: web.Request, _) -> web.Response:
     username = request.match_info['user']
 
     user = await db.select_and_fetchone(
-        '''
+        """
 SELECT id, username, login_id, state, is_developer, is_service_account, hail_identity FROM users
 WHERE username = %s;
-''',
+""",
         (username,),
     )
     if user is None:
@@ -599,11 +597,11 @@ async def _delete_user(db: Database, username: str, id: Optional[str]):
         where_args.append(id)
 
     n_rows = await db.execute_update(
-        f'''
+        f"""
 UPDATE users
 SET state = 'deleting'
 WHERE {' AND '.join(where_conditions)};
-''',
+""",
         where_args,
     )
 
@@ -743,11 +741,11 @@ async def get_userinfo_from_login_id_or_hail_identity_id(
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 WHERE (users.login_id = %s OR users.hail_identity_uid = %s) AND users.state = 'active'
-''',
+""",
             (login_id_or_hail_idenity_uid, login_id_or_hail_idenity_uid),
         )
     ]
@@ -767,12 +765,12 @@ async def get_userinfo_from_hail_session_id(request: web.Request, session_id: st
     users = [
         x
         async for x in db.select_and_fetchall(
-            '''
+            """
 SELECT users.*
 FROM users
 INNER JOIN sessions ON users.id = sessions.user_id
 WHERE users.state = 'active' AND sessions.session_id = %s AND (ISNULL(sessions.max_age_secs) OR (NOW() < TIMESTAMPADD(SECOND, sessions.max_age_secs, sessions.created)));
-''',
+""",
             session_id,
             'get_userinfo',
         )
@@ -812,13 +810,20 @@ class AppKeys:
     HAILCTL_CLIENT_CONFIG = web.AppKey('hailctl_client_config', dict)
     K8S_CLIENT = web.AppKey('k8s_client', kubernetes_asyncio.client.CoreV1Api)
     K8S_CACHE = web.AppKey('k8s_cache', K8sCache)
+    EXIT_STACK = web.AppKey('exit_stack', AsyncExitStack)
 
 
 async def on_startup(app):
+    exit_stack = AsyncExitStack()
+    app[AppKeys.EXIT_STACK] = exit_stack
+
     db = Database()
     await db.async_init(maxsize=50)
+    exit_stack.push_async_callback(db.async_close)
     app[AppKeys.DB] = db
+
     app[AppKeys.CLIENT_SESSION] = httpx.client_session()
+    exit_stack.push_async_callback(app[AppKeys.CLIENT_SESSION].close)
 
     credentials_file = '/auth-oauth2-client-secret/client_secret.json'
     if CLOUD == 'gcp':
@@ -832,14 +837,13 @@ async def on_startup(app):
 
     kubernetes_asyncio.config.load_incluster_config()
     app[AppKeys.K8S_CLIENT] = kubernetes_asyncio.client.CoreV1Api()
+    exit_stack.push_async_callback(app[AppKeys.K8S_CLIENT].api_client.rest_client.pool_manager.close)
+
     app[AppKeys.K8S_CACHE] = K8sCache(app[AppKeys.K8S_CLIENT])
 
 
 async def on_cleanup(app):
-    async with AsyncExitStack() as cleanup:
-        cleanup.push_async_callback(app[AppKeys.K8S_CLIENT].api_client.rest_client.pool_manager.close)
-        cleanup.push_async_callback(app[AppKeys.DB].async_close)
-        cleanup.push_async_callback(app[AppKeys.CLIENT_SESSION].close)
+    await app[AppKeys.EXIT_STACK].aclose()
 
 
 class AuthAccessLogger(AccessLogger):
@@ -879,6 +883,8 @@ async def auth_check_csrf_token(request: web.Request, handler: AIOHTTPHandler):
 
 
 def run():
+    uvloopx.install()
+
     install_profiler_if_requested('auth')
 
     app = web.Application(middlewares=[auth_check_csrf_token, monitor_endpoints_middleware])
@@ -898,5 +904,5 @@ def run():
         host='0.0.0.0',
         port=443,
         access_log_class=AuthAccessLogger,
-        ssl_context=internal_server_ssl_context(),
+        ssl_context=deploy_config.server_ssl_context(),
     )