Merge remote-tracking branch 'hi/main' into HEAD

.pre-commit-config.yaml

@@ -3,6 +3,12 @@ default_install_hook_types: [pre-commit, pre-push]
 repos:
   - repo: local
     hooks:
+      - id: ruff
+        name: ruff
+        entry: ruff --fix --show-fixes
+        language: system
+        types: [python]
+        require_serial: true
       - id: pyright
         name: pyright
         entry: pyright

auth/pinned-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --output-file=hail/auth/pinned-requirements.txt hail/auth/requirements.txt

batch/Dockerfile.worker

@@ -20,12 +20,15 @@ RUN echo "APT::Acquire::Retries \"5\";" > /etc/apt/apt.conf.d/80-retries && \
     hail-apt-get-install fuse gcsfuse=0.41.12
 
 {% elif global.cloud == "azure" %}
+RUN apt-get update && \
+    hail-apt-get-install libreadline8
+
 # https://github.com/Azure/azure-storage-fuse/issues/603
 RUN hail-apt-get-install ca-certificates pkg-config libfuse-dev cmake libcurl4-gnutls-dev libgnutls28-dev uuid-dev libgcrypt20-dev && \
-    curl -LO https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb && \
+    curl -LO https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb && \
     dpkg -i packages-microsoft-prod.deb && \
     apt-get update && \
-    hail-apt-get-install blobfuse
+    hail-apt-get-install blobfuse2
 
 {% else %}
 RUN echo "!!! UNEXPECTED CLOUD {{global.cloud}} !!!" && exit 1

batch/az-create-worker-image.sh

@@ -15,7 +15,7 @@ popd
 SHARED_GALLERY_NAME="${RESOURCE_GROUP}_batch"
 BUILD_IMAGE_RESOURCE_GROUP="${RESOURCE_GROUP}-build-batch-worker-image"
 VM_NAME=build-batch-worker-image
-WORKER_VERSION=0.0.12
+WORKER_VERSION=0.0.13
 
 USERNAME=$(whoami)
 
@@ -45,7 +45,7 @@ echo "Creating $VM_NAME VM..."
 IP=$(az vm create \
     --resource-group $BUILD_IMAGE_RESOURCE_GROUP \
     --name $VM_NAME \
-    --image UbuntuLTS \
+    --image Ubuntu2204 \
     --generate-ssh-keys \
     --public-ip-sku Standard \
     --assign-identity ${BATCH_WORKER_IDENTITY} \
@@ -81,7 +81,7 @@ az vm generalize \
     --name $VM_NAME
 
 az sig image-version delete \
-    --gallery-image-definition batch-worker \
+    --gallery-image-definition batch-worker-22-04 \
     --gallery-name ${SHARED_GALLERY_NAME} \
     --resource-group ${RESOURCE_GROUP} \
     --gallery-image-version ${WORKER_VERSION} || true
@@ -91,7 +91,7 @@ echo "Creating image..."
 az sig image-version create \
     --resource-group ${RESOURCE_GROUP} \
     --gallery-name ${SHARED_GALLERY_NAME} \
-    --gallery-image-definition batch-worker \
+    --gallery-image-definition batch-worker-22-04 \
     --gallery-image-version ${WORKER_VERSION} \
     --target-regions ${LOCATION} \
     --replica-count 1 \

batch/batch/cloud/azure/driver/create_instance.py

@@ -272,7 +272,6 @@ def create_vm_config(
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v /var/run/netns:/var/run/netns:shared \
 -v /usr/bin/docker:/usr/bin/docker \
--v /usr/sbin/xfs_quota:/usr/sbin/xfs_quota \
 -v /batch:/batch:shared \
 -v /logs:/logs \
 -v /global-config:/global-config \
@@ -290,6 +289,7 @@ def create_vm_config(
 --cap-add SYS_ADMIN \
 --security-opt apparmor:unconfined \
 --network host \
+--cgroupns host \
 $BATCH_WORKER_IMAGE \
 python3 -u -m batch.worker.worker >worker.log 2>&1
 
@@ -427,7 +427,7 @@ def create_vm_config(
                 'imageReference': {
                     'value': {
                         'id': f'/subscriptions/{subscription_id}/resourceGroups/{resource_group}/providers/'
-                        f'Microsoft.Compute/galleries/{resource_group}_batch/images/batch-worker/versions/0.0.12'
+                        f'Microsoft.Compute/galleries/{resource_group}_batch/images/batch-worker-22-04/versions/0.0.13'
                     }
                 },
                 'workspaceName': {

batch/batch/cloud/azure/worker/worker_api.py

@@ -92,7 +92,8 @@ async def _mount_cloudfuse(
             options.append('ro')
 
         await check_exec_output(
-            'blobfuse',
+            'blobfuse2',
+            'mountv1',
             mount_base_path_data,
             f'--tmp-path={mount_base_path_tmp}',
             f'--config-file={fuse_credentials_path}',

batch/batch/cloud/gcp/driver/create_instance.py

@@ -128,7 +128,7 @@ def scheduling() -> dict:
                 'boot': True,
                 'autoDelete': True,
                 'initializeParams': {
-                    'sourceImage': f'projects/{project}/global/images/batch-worker-12',
+                    'sourceImage': f'projects/{project}/global/images/batch-worker-13',
                     'diskType': f'projects/{project}/zones/{zone}/diskTypes/pd-ssd',
                     'diskSizeGb': str(boot_disk_size_gb),
                 },
@@ -380,6 +380,7 @@ def scheduling() -> dict:
 --cap-add SYS_ADMIN \
 --security-opt apparmor:unconfined \
 --network host \
+--cgroupns host \
 $BATCH_WORKER_IMAGE \
 python3 -u -m batch.worker.worker >worker.log 2>&1
 

batch/batch/cloud/gcp/instance_config.py

@@ -151,8 +151,8 @@ def from_dict(data: dict) -> 'GCPSlimInstanceConfig':
                 GCPStaticSizedDiskResource('disk/pd-ssd/1', boot_disk_size_gb),
                 data_disk_resource,
                 GCPDynamicSizedDiskResource('disk/pd-ssd/1'),
-                GCPIPFeeResource('service-fee/1'),
-                GCPServiceFeeResource('ip-fee/1024/1'),
+                GCPServiceFeeResource('service-fee/1'),
+                GCPIPFeeResource('ip-fee/1024/1'),
                 GCPSupportLogsSpecsAndFirewallFees('gcp-support-logs-specs-and-firewall-fees/1'),
             ]
         else:

batch/gcp-create-worker-image.sh

@@ -14,7 +14,7 @@ PROJECT=$(get_global_config_field gcp_project $NAMESPACE)
 ZONE=$(get_global_config_field gcp_zone $NAMESPACE)
 DOCKER_ROOT_IMAGE=$(get_global_config_field docker_root_image $NAMESPACE)
 
-WORKER_IMAGE_VERSION=12
+WORKER_IMAGE_VERSION=13
 
 if [ "$NAMESPACE" == "default" ]; then
     WORKER_IMAGE=batch-worker-${WORKER_IMAGE_VERSION}
@@ -24,7 +24,7 @@ else
     BUILDER=build-batch-worker-$NAMESPACE-image
 fi
 
-UBUNTU_IMAGE=ubuntu-minimal-2004-focal-v20230725
+UBUNTU_IMAGE=ubuntu-minimal-2204-jammy-v20230726
 
 create_build_image_instance() {
     gcloud -q compute --project ${PROJECT} instances delete \

batch/pinned-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --output-file=hail/batch/pinned-requirements.txt hail/batch/requirements.txt
@@ -64,7 +64,7 @@ multidict==6.0.4
     #   -c hail/batch/../web_common/pinned-requirements.txt
     #   aiohttp
     #   yarl
-numpy==1.24.4
+numpy==1.25.2
     # via
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
@@ -74,11 +74,11 @@ packaging==23.1
     #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   plotly
-pandas==2.0.3
+pandas==2.1.0
     # via
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
-plotly==5.16.0
+plotly==5.16.1
     # via
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   -r hail/batch/requirements.txt
@@ -90,7 +90,6 @@ python-dateutil==2.8.2
     #   pandas
 pytz==2023.3
     # via
-    #   -c hail/batch/../hail/python/dev/pinned-requirements.txt
     #   -c hail/batch/../hail/python/pinned-requirements.txt
     #   pandas
 six==1.16.0

batch/test/test_batch.py

@@ -1111,7 +1111,7 @@ def test_verify_no_access_to_metadata_server(client: BatchClient):
     status = j.wait()
     assert status['state'] == 'Failed', str((status, b.debug_info()))
     job_log = j.log()
-    assert "Connection timed out" in job_log['main'], str((job_log, b.debug_info()))
+    assert "Connection timeout" in job_log['main'], str((job_log, b.debug_info()))
 
 
 def test_submit_batch_in_job(client: BatchClient, remote_tmpdir: str):

batch/test/utils.py

@@ -6,7 +6,7 @@
 import hailtop.batch_client.client as bc
 from hailtop import pip_version
 
-DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:20.04')
+DOCKER_ROOT_IMAGE = os.environ.get('DOCKER_ROOT_IMAGE', 'ubuntu:22.04')
 HAIL_GENETICS_HAIL_IMAGE = os.environ.get('HAIL_GENETICS_HAIL_IMAGE', f'hailgenetics/hail:{pip_version()}')
 
 

benchmark/Makefile

@@ -31,7 +31,7 @@ cleanup_image:
 BENCHMARK_DOCKER_TAG ?= $(shell whoami)
 BECNHMARK_IMAGE_REPOSITORY ?= us-docker.pkg.dev/broad-ctsa/hail-benchmarks
 BENCHMARK_REPO_BASE = $(BECNHMARK_IMAGE_REPOSITORY)/$(BENCHMARK_DOCKER_TAG)
-DOCKER_ROOT_IMAGE := ubuntu:20.04
+DOCKER_ROOT_IMAGE := ubuntu:22.04
 
 ifndef HAIL_WHEEL
 image_sha:

build.yaml

@@ -2781,7 +2781,7 @@ steps:
           --from-literal=github_context="ci-test" \
           --from-literal=storage_uri="{{ global.test_storage_uri }}" \
           --from-literal=deploy_steps="[]" \
-          --from-literal=watched_branches="[[\"hail-ci-test/ci-test-{{create_ci_test_repo.token}}:master\", true, true]]" \
+          --from-literal=watched_branches="[[\"hail-ci-test/ci-test-{{create_ci_test_repo.token}}:main\", true, true]]" \
           --save-config --dry-run=client -o yaml \
         | kubectl -n {{ default_ns.name }} apply -f -
     serviceAccount:

ci/ci/github.py

@@ -202,6 +202,7 @@ def __init__(self, exception: BaseException, attributes: Dict[str, str]):
 HIGH_PRIORITY = 'prio:high'
 STACKED_PR = 'stacked PR'
 WIP = 'WIP'
+DO_NOT_TEST = 'do-not-test'
 
 DO_NOT_MERGE = {STACKED_PR, WIP}
 
@@ -308,6 +309,9 @@ def build_succeeding_on_all_platforms(self):
     def build_failed_on_at_least_one_platform(self):
         return any(gh_status == GithubStatus.FAILURE for gh_status in self.last_known_github_status.values())
 
+    def testable(self):
+        return DO_NOT_TEST not in self.labels
+
     def merge_priority(self):
         # passed > unknown > failed
         if self.build_succeeding_on_all_platforms():
@@ -637,6 +641,9 @@ async def _heal(self, batch_client, db: Database, on_deck, gh):
         if not await self.authorized(db):
             return
 
+        if not self.testable():
+            return
+
         if not self.batch or (on_deck and self.batch.attributes['target_sha'] != self.target_branch.sha):
             if on_deck or self.target_branch.n_running_batches < MAX_CONCURRENT_PR_BATCHES:
                 self.target_branch.n_running_batches += 1

ci/pinned-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --output-file=hail/ci/pinned-requirements.txt hail/ci/requirements.txt
@@ -22,7 +22,7 @@ charset-normalizer==3.2.0
     #   -c hail/ci/../hail/python/pinned-requirements.txt
     #   -c hail/ci/../web_common/pinned-requirements.txt
     #   requests
-click==8.1.6
+click==8.1.7
     # via
     #   -c hail/ci/../gear/pinned-requirements.txt
     #   -c hail/ci/../hail/python/dev/pinned-requirements.txt

docker/hail-ubuntu/Dockerfile

@@ -1,4 +1,4 @@
-FROM {{ global.docker_prefix }}/ubuntu:focal-20221019
+FROM {{ global.docker_prefix }}/ubuntu:jammy-20230624
 ENV LANG C.UTF-8
 ENV DEBIAN_FRONTEND noninteractive
 COPY retry /bin/retry
@@ -16,9 +16,9 @@ RUN chmod 755 /bin/retry && \
     hail-apt-get-install curl gpg jq rsync && \
     curl 'https://keyserver.ubuntu.com/pks/lookup?search=0xF23C5A6CF475977595C89F51BA6932366A755776&hash=on&exact=on&options=mr&op=get' \
          | gpg --dearmor > /usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg && \
-    echo 'deb [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \
+    echo 'deb [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main' \
          >> /etc/apt/sources.list && \
-    echo 'deb-src [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal main' \
+    echo 'deb-src [signed-by=/usr/share/keyrings/deadsnakes-ppa-archive-keyring.gpg] http://ppa.launchpad.net/deadsnakes/ppa/ubuntu jammy main' \
          >> /etc/apt/sources.list && \
     hail-apt-get-install python3.9-minimal python3.9-dev python3.9-distutils gcc g++ && \
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \

docker/hailgenetics/mirror_images.sh

@@ -10,11 +10,8 @@ copy_if_not_present() {
     if ! skopeo inspect "docker://docker.io/$1";
     then
         echo "$1 does not exist yet, doing nothing"
-    elif skopeo inspect "docker://$2";
-    then
-      echo "$2 already exists, doing nothing"
     else
-      echo "$2 does not exist, copying $1 to $2"
+      echo "copying $1 to $2"
       copy_image $1 $2
     fi
 }

docker/hailgenetics/python-dill/Dockerfile

@@ -1,6 +1,6 @@
 ARG PYTHON_VERSION
 FROM python:${PYTHON_VERSION}
-RUN pip install --upgrade --no-cache-dir dill numpy scipy scikit-learn && \
+RUN pip install --upgrade --no-cache-dir 'dill>=0.3.6' numpy scipy scikit-learn && \
     python3 -m pip check && \
     apt-get update && \
     apt-get install -y \

docker/third-party/images.txt

@@ -13,6 +13,6 @@ python:3.10-slim
 redis:6.0.6-alpine
 ubuntu:18.04
 ubuntu:19.04
-ubuntu:20.04
+ubuntu:22.04
 ubuntu:bionic-20200921
 ubuntu:focal-20221019

gear/pinned-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile --output-file=hail/gear/pinned-requirements.txt hail/gear/requirements.txt
@@ -57,7 +57,7 @@ charset-normalizer==3.2.0
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   aiohttp
     #   requests
-click==8.1.6
+click==8.1.7
     # via
     #   -c hail/gear/../hail/python/dev/pinned-requirements.txt
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
@@ -129,7 +129,7 @@ multidict==6.0.4
     #   -c hail/gear/../hail/python/pinned-requirements.txt
     #   aiohttp
     #   yarl
-orjson==3.9.4
+orjson==3.9.5
     # via
     #   -c hail/gear/../hail/python/hailtop/pinned-requirements.txt
     #   -c hail/gear/../hail/python/pinned-requirements.txt