Upstream 0.2.100

batch/batch/front_end/front_end.py

@@ -72,7 +72,7 @@
 from ..globals import BATCH_FORMAT_VERSION, HTTP_CLIENT_MAX_SIZE
 from ..inst_coll_config import InstanceCollectionConfigs
 from ..spec_writer import SpecWriter
-from ..utils import query_billing_projects
+from ..utils import query_billing_projects, unavailable_if_frozen
 from .validate import ValidationError, validate_and_clean_jobs, validate_batch
 
 # import uvloop
@@ -802,10 +802,6 @@ def check_service_account_permissions(user, sa):
 async def create_jobs(request: aiohttp.web.Request, userdata: dict):
     app = request.app
 
-    if app['frozen']:
-        log.info('ignoring batch create request; batch is frozen')
-        raise web.HTTPServiceUnavailable()
-
     batch_id = int(request.match_info['batch_id'])
     job_specs = await request.json()
     return await _create_jobs(userdata, job_specs, batch_id, 1, app)
@@ -1241,10 +1237,6 @@ async def create_batch_fast(request, userdata):
     app = request.app
     db: Database = app['db']
 
-    if app['frozen']:
-        log.info('ignoring batch create request; batch is frozen')
-        raise web.HTTPServiceUnavailable()
-
     user = userdata['username']
     batch_and_bunch = await request.json()
     batch_spec = batch_and_bunch['batch']
@@ -1268,10 +1260,6 @@ async def create_batch(request, userdata):
     app = request.app
     db: Database = app['db']
 
-    if app['frozen']:
-        log.info('ignoring batch create jobs request; batch is frozen')
-        raise web.HTTPServiceUnavailable()
-
     batch_spec = await request.json()
     id = await _create_batch(batch_spec, userdata, db)
     n_jobs = batch_spec['n_jobs']
@@ -1508,10 +1496,6 @@ async def close_batch(request, userdata):
     app = request.app
     db: Database = app['db']
 
-    if app['frozen']:
-        log.info('ignoring batch close request; batch is frozen')
-        raise web.HTTPServiceUnavailable()
-
     record = await db.select_and_fetchone(
         '''
 SELECT 1 FROM batches
@@ -2090,6 +2074,8 @@ async def ui_get_billing(request, userdata):
     ]
     billing_by_project_user.sort(key=lambda record: (record['billing_project'], record['user']))
 
+    total_cost = cost_str(sum([record['cost'] for record in billing]))
+
     page_context = {
         'billing_by_project': billing_by_project,
         'billing_by_user': billing_by_user,
@@ -2098,6 +2084,7 @@ async def ui_get_billing(request, userdata):
         'end': end,
         'is_developer': is_developer,
         'user': userdata['username'],
+        'total_cost': total_cost,
     }
     return await render_template('batch', request, userdata, 'billing.html', page_context)
 
@@ -2592,7 +2579,9 @@ async def on_cleanup(app):
 
 
 def run():
-    app = web.Application(client_max_size=HTTP_CLIENT_MAX_SIZE, middlewares=[monitor_endpoints_middleware])
+    app = web.Application(
+        client_max_size=HTTP_CLIENT_MAX_SIZE, middlewares=[unavailable_if_frozen, monitor_endpoints_middleware]
+    )
     setup_aiohttp_session(app)
 
     setup_aiohttp_jinja2(app, 'batch.front_end')

batch/batch/front_end/templates/billing.html

@@ -13,7 +13,7 @@ <h1>Billing</h1>
                    placeholder="MM/DD/YYYY"
                    {% endif %}
             ><br>
-            <label for="end">End:</label>
+            <label for="end">End (inclusive):</label>
             <input style="vertical-align:text-bottom;" name="end" size=30 type="text"
                    {% if end is not none %}
                    value = "{{ end }}"
@@ -34,6 +34,11 @@ <h1>Billing</h1>
         </div>
     </div>
 
+    <h2>Total Cost</h2>
+    <ul>
+        <li>{{ total_cost }}</li>
+    </ul>
+
     {% if is_developer %}
     <h2>Cost by Billing Project</h2>
     <div class='flex-col' style="overflow: auto;">

batch/batch/utils.py

@@ -13,6 +13,13 @@
 log = logging.getLogger('utils')
 
 
+@web.middleware
+async def unavailable_if_frozen(request: web.Request, handler):
+    if request.method in ("POST", "PATCH") and request.app['frozen']:
+        raise web.HTTPServiceUnavailable()
+    return await handler(request)
+
+
 def authorization_token(request):
     auth_header = request.headers.get('Authorization')
     if not auth_header:

batch/sql/add-jobs-update-id-index.sql

@@ -0,0 +1 @@
+CREATE INDEX `jobs_batch_id_update_id` ON `jobs` (`batch_id`, `update_id`);

batch/sql/estimated-current.sql

@@ -254,6 +254,7 @@ CREATE TABLE IF NOT EXISTS `jobs` (
 ) ENGINE = InnoDB;
 CREATE INDEX `jobs_batch_id_state_always_run_inst_coll_cancelled` ON `jobs` (`batch_id`, `state`, `always_run`, `inst_coll`, `cancelled`);
 CREATE INDEX `jobs_batch_id_state_always_run_cancelled` ON `jobs` (`batch_id`, `state`, `always_run`, `cancelled`);
+CREATE INDEX `jobs_batch_id_update_id` ON `jobs` (`batch_id`, `update_id`);
 
 CREATE TABLE IF NOT EXISTS `batch_bunches` (
   `batch_id` BIGINT NOT NULL,

batch/sql/no_dev_standing_workers_by_default.py

@@ -0,0 +1,22 @@
+import os
+import asyncio
+from gear import Database
+
+
+async def main():
+    if os.environ['HAIL_SCOPE'] != 'dev':
+        return
+
+    db = Database()
+    await db.async_init()
+
+    await db.execute_update(
+        '''
+UPDATE pools
+SET enable_standing_worker = 0
+'''
+    )
+
+
+loop = asyncio.get_event_loop()
+loop.run_until_complete(main())

build.yaml

@@ -2097,6 +2097,12 @@ steps:
       - name: cleanup-add-batch-updates
         script: /io/sql/cleanup-add-batch-updates.sql
         online: true
+      - name: add-jobs-update-id-index
+        script: /io/sql/add-jobs-update-id-index.sql
+        online: true
+      - name: no-dev-standing-workers-by-default
+        script: /io/sql/no_dev_standing_workers_by_default.py
+        online: true
     inputs:
       - from: /repo/batch/sql
         to: /io/sql
@@ -2707,14 +2713,14 @@ steps:
       - create_ci_test_repo
   - kind: buildImage2
     name: hailgenetics_hail_image
-    dockerFile: /io/docker/hail/Dockerfile
-    contextPath: /io/docker/hail/
+    dockerFile: /io/docker/hailgenetics/hail/Dockerfile
+    contextPath: /io/docker/hailgenetics/hail
     publishAs: hailgenetics/hail
     inputs:
-      - from: /repo/docker/hail
-        to: /io/docker/hail
+      - from: /repo/docker/hailgenetics/hail
+        to: /io/docker/hailgenetics/hail
       - from: /just-wheel/wheel-container.tar
-        to: /io/docker/hail/wheel-container.tar
+        to: /io/docker/hailgenetics/hail/wheel-container.tar
     dependsOn:
       - merge_code
       - build_hail_jar_and_wheel_only

ci/buildkit/convert-cloud-credentials-to-docker-auth-config

@@ -13,4 +13,11 @@ elif [ ! -z "${AZURE_APPLICATION_CREDENTIALS}" ]; then
 fi
 
 USER_PASS=$(echo -n "$USERNAME:$PASSWORD" | base64 | tr -d \\n)
-echo '{"auths": { "'$REGISTRY'": { "auth": "'$USER_PASS'"}}}' > $HOME/.docker/config.json
+
+registry_auths='{}'
+for registry in ${REGISTRIES:=$REGISTRY}
+do
+    registry_auths=$(echo $registry_auths | jq --arg registry "$registry" --arg userpass "$USER_PASS" '. + { ($registry): { "auth": $userpass }}')
+done
+
+echo '{"auths": '$registry_auths'}' > $HOME/.docker/config.json

docker/copy_image.sh

@@ -0,0 +1,13 @@
+if command -v skopeo
+then
+    copy_image() {
+        skopeo copy --override-os linux --override-arch amd64 docker://docker.io/$1 docker://$2
+    }
+else
+    echo Could not find skopeo, falling back to docker which will be slower.
+    copy_image() {
+        docker pull $1
+        docker tag $1 $2
+        docker push $2
+    }
+fi

docker/hailgenetics/Makefile

@@ -0,0 +1,11 @@
+include ../../config.mk
+
+.PHONY: publish-python-dill
+
+publish-python-dill:
+	DOCKER_PREFIX=$(DOCKER_PREFIX) bash python-dill/push.sh
+
+mirror-dockerhub-images:
+	DOCKER_PREFIX=$(DOCKER_PREFIX) \
+	HAIL_PIP_VERSION=$(shell cat ../../hail/python/hail/hail_pip_version) \
+	./mirror_images.sh

docker/hailgenetics/mirror_images.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -ex
+
+source ../copy_image.sh
+
+if [[ -z "${DOCKER_PREFIX}" ]];
+then
+    echo "Env variable DOCKER_PREFIX must be set"
+    exit 1
+fi
+
+if [[ -z "${HAIL_PIP_VERSION}" ]];
+then
+    echo "Env variable HAIL_PIP_VERSION must be set"
+    exit 1
+fi
+
+python_dill_images=(
+    "python-dill:3.7"
+    "python-dill:3.7-slim"
+    "python-dill:3.8"
+    "python-dill:3.8-slim"
+    "python-dill:3.9"
+    "python-dill:3.9-slim"
+    "python-dill:3.10"
+    "python-dill:3.10-slim"
+)
+
+for image in "${python_dill_images[@]}"
+do
+    copy_image "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
+done
+
+pip_release_images=(
+    "hail:${HAIL_PIP_VERSION}"
+    "genetics:${HAIL_PIP_VERSION}"
+)
+for image in "${pip_release_images[@]}"
+do
+    copy_image "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
+done

docker/python-dill/Dockerfile → docker/hailgenetics/python-dill/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:@PYTHON_VERSION@
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}
 RUN pip install --upgrade --no-cache-dir dill numpy scipy sklearn && \
     python3 -m pip check && \
     apt-get update && \

docker/hailgenetics/python-dill/push.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+for version in 3.7 3.7-slim 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim
+do
+    public=hailgenetics/python-dill:$version
+
+    DOCKER_BUILDKIT=1 docker build \
+        --build-arg PYTHON_VERSION=$version \
+        --file Dockerfile.out \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        --tag ${public} \
+        .
+
+    time DOCKER_BUILDKIT=1 docker push ${public}
+done

docker/linux-pinned-requirements.txt

@@ -173,7 +173,7 @@ greenlet==1.1.2
     # via gevent
 grpc-google-iam-v1==0.12.4
     # via google-cloud-logging
-grpcio==1.47.0
+grpcio==1.48.1
     # via
     #   google-api-core
     #   googleapis-common-protos

docker/third-party/copy_images.sh

@@ -2,6 +2,8 @@
 
 set -ex
 
+source ../copy_image.sh
+
 images=$(cat images.txt)
 
 if [ -z "${DOCKER_PREFIX}" ]
@@ -10,21 +12,6 @@ then
     exit 1
 fi
 
-if command -v skopeo
-then
-    copy_image() {
-        skopeo copy --override-os linux --override-arch amd64 docker://docker.io/$1 docker://$2
-    }
-else
-    echo Could not find skopeo, falling back to docker which will be slower.
-    copy_image() {
-        docker pull $1
-        docker tag $1 $2
-        docker push $2
-    }
-fi
-
-
 for image in ${images}
 do
     dest="${DOCKER_PREFIX}/${image}"

hail/Makefile

@@ -14,7 +14,7 @@ BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 SCALA_VERSION ?= 2.12.13
 SPARK_VERSION ?= 3.1.3
 HAIL_MAJOR_MINOR_VERSION := 0.2
-HAIL_PATCH_VERSION := 99
+HAIL_PATCH_VERSION := 100
 HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION)
 HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION)
 ELASTIC_MAJOR_VERSION ?= 8
@@ -262,7 +262,7 @@ install: $(WHEEL)
 
 .PHONY: install-on-cluster
 install-on-cluster: $(WHEEL)
-	sed '/^pyspark/d' python/requirements.txt | grep -v '^#' | xargs $(PIP) install -U
+	sed '/^pyspark/d' python/requirements.txt | grep -v '^#' | tr '\n' '\0' | xargs -0 $(PIP) install -U
 	-$(PIP) uninstall -y hail
 	$(PIP) install $(WHEEL) --no-deps
 

hail/python/dev/pinned-requirements.txt

@@ -404,7 +404,7 @@ types-urllib3==1.26.15
     # via
     #   -r python/dev/requirements.txt
     #   types-requests
-typing-extensions==4.2.0
+typing-extensions==4.3.0
     # via
     #   argon2-cffi
     #   astroid