Merge upstream changes

.gitignore

@@ -27,3 +27,8 @@ GTAGS
 *.dylib
 */hail.jar
 infra/.terraform.lock.hcl
+hail/python/hail/docs/experimental/hail.experimental.DB.rst
+hail/python/hailtop/batch/docs/api/
+web_common/web_common/static/css/
+website/docs.tar.gz
+website/website/static/css/

auth/auth/auth.py

@@ -8,7 +8,6 @@
 import google.auth.transport.requests
 import google.oauth2.id_token
 import google_auth_oauthlib.flow
-from hailtop.auth import async_get_userinfo
 from hailtop.config import get_deploy_config
 from hailtop.tls import internal_server_ssl_context
 from hailtop.hail_logging import AccessLogger
@@ -526,18 +525,7 @@ async def rest_logout(request, userdata):
     return web.Response(status=200)
 
 
-@routes.get('/api/v1alpha/userinfo')
-async def userinfo(request):
-    if 'Authorization' not in request.headers:
-        log.info('Authorization not in request.headers')
-        raise web.HTTPUnauthorized()
-
-    auth_header = request.headers['Authorization']
-    session_id = maybe_parse_bearer_header(auth_header)
-    if not session_id:
-        log.info('Bearer not in Authorization header')
-        raise web.HTTPUnauthorized()
-
+async def get_userinfo(request, session_id):
     # b64 encoding of 32-byte session ID is 44 bytes
     if len(session_id) != 44:
         log.info('Session id != 44 bytes')
@@ -554,18 +542,41 @@ async def userinfo(request):
     if len(users) != 1:
         log.info(f'Unknown session id: {session_id}')
         raise web.HTTPUnauthorized()
-    user = users[0]
+    return users[0]
+
+
+@routes.get('/api/v1alpha/userinfo')
+async def userinfo(request):
+    if 'Authorization' not in request.headers:
+        log.info('Authorization not in request.headers')
+        raise web.HTTPUnauthorized()
+
+    auth_header = request.headers['Authorization']
+    session_id = maybe_parse_bearer_header(auth_header)
+    if not session_id:
+        log.info('Bearer not in Authorization header')
+        raise web.HTTPUnauthorized()
+
+    return web.json_response(await get_userinfo(request, session_id))
+
 
-    return web.json_response(user)
+async def get_session_id(request):
+    if 'X-Hail-Internal-Authorization' in request.headers:
+        return maybe_parse_bearer_header(request.headers['X-Hail-Internal-Authorization'])
+
+    if 'Authorization' in request.headers:
+        return maybe_parse_bearer_header(request.headers['Authorization'])
+
+    session = await aiohttp_session.get_session(request)
+    return session.get('session_id')
 
 
 @routes.get('/api/v1alpha/verify_dev_credentials')
 async def verify_dev_credentials(request):
-    session = await aiohttp_session.get_session(request)
-    session_id = session.get('session_id')
+    session_id = await get_session_id(request)
     if not session_id:
         raise web.HTTPUnauthorized()
-    userdata = await async_get_userinfo(session_id=session_id)
+    userdata = await get_userinfo(request, session_id)
     is_developer = userdata is not None and userdata['is_developer'] == 1
     if not is_developer:
         raise web.HTTPUnauthorized()

batch/Dockerfile.worker

@@ -10,7 +10,7 @@ RUN hail-apt-get-install \
 COPY docker/hail-ubuntu/pip.conf /root/.config/pip/pip.conf
 COPY docker/hail-ubuntu/hail-pip-install /bin/hail-pip-install
 COPY docker/requirements.txt .
-RUN hail-pip-install -r requirements.txt pyspark==2.4.0
+RUN hail-pip-install -r requirements.txt pyspark==3.1.1
 
 ENV SPARK_HOME /usr/local/lib/python3.7/site-packages/pyspark
 ENV PATH "$PATH:$SPARK_HOME/sbin:$SPARK_HOME/bin"

batch/batch/batch.py

@@ -41,6 +41,7 @@ def _time_msecs_str(t):
 
     d = {
         'id': record['id'],
+        'user': record['user'],
         'billing_project': record['billing_project'],
         'token': record['token'],
         'state': state,
@@ -85,6 +86,8 @@ def job_record_to_dict(record, name):
         'batch_id': record['batch_id'],
         'job_id': record['job_id'],
         'name': name,
+        'user': record['user'],
+        'billing_project': record['billing_project'],
         'state': record['state'],
         'exit_code': exit_code,
         'duration': duration

batch/batch/front_end/front_end.py

@@ -222,7 +222,8 @@ async def _query_batch_jobs(request, batch_id):
         where_args.extend(args)
 
     sql = f'''
-SELECT jobs.*, batches.format_version, job_attributes.value AS name, SUM(`usage` * rate) AS cost
+SELECT jobs.*, batches.user, batches.billing_project,  batches.format_version,
+  job_attributes.value AS name, SUM(`usage` * rate) AS cost
 FROM jobs
 INNER JOIN batches ON jobs.batch_id = batches.id
 LEFT JOIN job_attributes
@@ -1150,7 +1151,7 @@ async def _get_job(app, batch_id, job_id):
     db: Database = app['db']
 
     record = await db.select_and_fetchone('''
-SELECT jobs.*, ip_address, format_version, SUM(`usage` * rate) AS cost
+SELECT jobs.*, user, billing_project, ip_address, format_version, SUM(`usage` * rate) AS cost
 FROM jobs
 INNER JOIN batches
   ON jobs.batch_id = batches.id
@@ -1252,28 +1253,31 @@ async def ui_get_job(request, userdata, batch_id):
     app = request.app
     job_id = int(request.match_info['job_id'])
 
-    job_status, attempts, job_log = await asyncio.gather(_get_job(app, batch_id, job_id),
-                                                         _get_attempts(app, batch_id, job_id),
-                                                         _get_job_log(app, batch_id, job_id))
+    job, attempts, job_log = await asyncio.gather(_get_job(app, batch_id, job_id),
+                                                  _get_attempts(app, batch_id, job_id),
+                                                  _get_job_log(app, batch_id, job_id))
 
-    job_status_status = job_status['status']
+    job['duration'] = humanize_timedelta_msecs(job['duration'])
+    job['cost'] = cost_str(job['cost'])
+
+    job_status = job['status']
     container_status_spec = dictfix.NoneOr({
         'name': str,
         'timing': {'pulling': dictfix.NoneOr({'duration': dictfix.NoneOr(Number)}),
                    'running': dictfix.NoneOr({'duration': dictfix.NoneOr(Number)})},
         'container_status': {'out_of_memory': False},
         'state': str})
-    job_status_status_spec = {
+    job_status_spec = {
         'container_statuses': {'input': container_status_spec,
                                'main': container_status_spec,
                                'output': container_status_spec}}
-    job_status_status = dictfix.dictfix(job_status_status, job_status_status_spec)
-    container_statuses = job_status_status['container_statuses']
+    job_status = dictfix.dictfix(job_status, job_status_spec)
+    container_statuses = job_status['container_statuses']
     step_statuses = [container_statuses['input'],
                      container_statuses['main'],
                      container_statuses['output']]
 
-    job_specification = job_status['spec']
+    job_specification = job['spec']
     if 'process' in job_specification:
         process_specification = job_specification['process']
         process_type = process_specification['type']
@@ -1289,11 +1293,12 @@ async def ui_get_job(request, userdata, batch_id):
     page_context = {
         'batch_id': batch_id,
         'job_id': job_id,
+        'job': job,
         'job_log': job_log,
         'attempts': attempts,
         'step_statuses': step_statuses,
         'job_specification': job_specification,
-        'job_status_str': json.dumps(job_status, indent=2)
+        'job_status_str': json.dumps(job, indent=2)
     }
     return await render_template('batch', request, userdata, 'job.html', page_context)
 

batch/batch/front_end/templates/batch.html

@@ -4,7 +4,28 @@
   <script src="{{ base_path }}/common_static/focus_on_keyup.js"></script>
 {% endblock %}
 {% block content %}
+
 <h1>Batch {{ batch['id'] }}</h1>
+
+<h2>Properties</h2>
+<ul>
+  <li>User: {{ batch['user'] }}</li>
+  <li>Billing Project: {{ batch['billing_project'] }}</li>
+  <li>Time Created: {% if 'time_created' in batch and batch['time_created'] is not none %}{{ batch['time_created'] }}{% endif %}</li>
+  <li>Time Closed: {% if 'time_closed' in batch and batch['time_closed'] is not none %}{{ batch['time_closed'] }}{% endif %}</li>
+  <li>Time Completed: {% if 'time_completed' in batch and batch['time_completed'] is not none %}{{ batch['time_completed'] }}{% endif %}</li>
+  <li>Total Jobs: {{ batch['n_jobs'] }}</li>
+  <ul>
+    <li>Pending Jobs: {{ batch['n_jobs'] - batch['n_completed'] }}</li>
+    <li>Succeeded Jobs: {{ batch['n_succeeded'] }}</li>
+    <li>Failed Jobs: {{ batch['n_failed'] }}</li>
+    <li>Cancelled Jobs: {{ batch['n_cancelled'] }}</li>
+  </ul>
+  <li>Duration: {% if 'duration' in batch and batch['duration'] is not none %}{{ batch['duration'] }}{% endif %}</li>
+  <li>Cost: {% if 'cost' in batch and batch['cost'] is not none %}{{ batch['cost'] }}{% endif %}</li>
+</ul>
+
+<h2>Attributes</h2>
 {% if 'attributes' in batch %}
 {% for name, value in batch['attributes'].items() %}
 <p>{{ name }}: {{ value }}</p>
@@ -64,7 +85,7 @@ <h2>Jobs</h2>
       <tbody>
         {% for job in batch['jobs'] %}
         <tr>
-          <td class="numeric-cell">
+          <td class="numeric-cell" onClick="document.location.href='{{ base_path }}/batches/{{ job['batch_id'] }}/jobs/{{ job['job_id'] }}';">
             <a href="{{ base_path }}/batches/{{ job['batch_id'] }}/jobs/{{ job['job_id'] }}">{{ job['job_id'] }}</a>
           </td>
           <td>

batch/batch/front_end/templates/batches.html

@@ -52,6 +52,8 @@ <h1>Batches</h1>
 	<thead>
 	  <tr>
 	    <th>ID</th>
+	    <th>User</th>
+	    <th>Billing Project</th>
 	    <th>Name</th>
 	    <th>Submitted</th>
 	    <th>Completed</th>
@@ -68,7 +70,11 @@ <h1>Batches</h1>
 	<tbody>
 	  {% for batch in batches %}
 	  <tr>
-	    <td class="numeric-cell"><a href="{{ base_path }}/batches/{{ batch['id'] }}">{{ batch['id'] }}</a></td>
+	    <td class="numeric-cell" onClick="document.location.href='{{ base_path }}/batches/{{ batch['id'] }}';">
+              <a href="{{ base_path }}/batches/{{ batch['id'] }}">{{ batch['id'] }}</a>
+	    </td>
+	    <td>{{ batch['user'] }}</td>
+	    <td>{{ batch['billing_project'] }}</td>
 	    <td>
 	      {% if 'attributes' in batch and 'name' in batch['attributes'] and batch['attributes']['name'] is not none %}
 	      {{ batch['attributes']['name'] }}

batch/batch/front_end/templates/job.html

@@ -3,6 +3,18 @@
 {% block content %}
 <h1>Batch {{ batch_id }} Job {{ job_id }}</h1>
 
+<h2>Properties</h2>
+<ul>
+  <li><a href="{{ base_path }}/batches/{{ batch_id }}">Batch ID: {{ batch_id }}</a></li>
+  <li>Job ID: {{ job_id }}</li>
+  <li>User: {{ job['user'] }} </li>
+  <li>Billing Project: {{ job['billing_project'] }}</li>
+  <li>State: {{ job['state'] }}</li>
+  <li>Exit Code: {% if 'exit_code' in job and job['exit_code'] is not none %}{{ job['exit_code'] }}{% endif %}</li>
+  <li>Duration: {% if 'duration' in job and job['duration'] is not none %}{{ job['duration'] }}{% endif %}</li>
+  <li>Cost: {% if 'cost' in job and job['cost'] is not none %}{{ job['cost'] }}{% endif %}</li>
+</ul>
+
 <h2>Attempts</h2>
 {% if attempts %}
 <table class="data-table">

batch/test/test_dag.py

@@ -156,6 +156,7 @@ def test():
         callback_body.pop('duration')
         assert (callback_body == {
             'id': b.id,
+            'user': 'test',
             'billing_project': 'test',
             'token': token,
             'state': 'success',

benchmark-service/Dockerfile.test

@@ -1,6 +1,3 @@
 FROM {{ service_base_image.image }}
 
 COPY benchmark-service/test/ /test/
-RUN python3 -m pip install --no-cache-dir \
-  pytest-instafail==0.4.1 \
-  pytest-asyncio==0.10.0

benchmark-service/test/test_update_commits.py

@@ -9,14 +9,13 @@
 from hailtop.httpx import client_session
 import hailtop.utils as utils
 
-pytestmark = pytest.mark.asyncio
-
 logging.basicConfig(level=logging.INFO)
 log = logging.getLogger(__name__)
 
 sha = 'd626f793ad700c45a878d192652a0378818bbd8b'
 
 
+@pytest.mark.asyncio
 async def test_update_commits():
     deploy_config = get_deploy_config()
     headers = service_auth_headers(deploy_config, 'benchmark')

build.yaml

@@ -684,26 +684,6 @@ steps:
        to: /cluster-tests.tar.gz
    dependsOn:
     - hail_build_image
- - kind: runImage
-   name: build_hail_spark3
-   image:
-     valueFrom: hail_build_image.image
-   resources:
-     memory: "7.5G"
-     cpu: "4"
-   script: |
-     set -ex
-     cd /
-     rm -rf repo
-     mkdir repo
-     cd repo
-     {{ code.checkout_script }}
-     cd hail
-     time retry ./gradlew --version
-     export SPARK_VERSION="3.0.1" SCALA_VERSION="2.12.12"
-     time retry make jars python-version-info wheel
-   dependsOn:
-     - hail_build_image
  - kind: buildImage
    name: batch_worker_image
    dockerFile: batch/Dockerfile.worker
@@ -2830,6 +2810,8 @@ steps:
      mkdir -p ./ci/test ./hail/python
      cp /repo/hail/ci/test/resources/build.yaml ./
      cp -R /repo/hail/ci/test/resources ./ci/test/
+     cp /repo/hail/tls/Dockerfile ./ci/test/resources/Dockerfile.certs
+     cp /repo/hail/tls/create_certs.py ./ci/test/resources/
      cp /repo/hail/pylintrc ./
      cp /repo/hail/setup.cfg ./
      cp -R /repo/hail/docker ./
@@ -3289,7 +3271,7 @@ steps:
    script: |
      set -ex
      gcloud auth activate-service-account --key-file=/secrets/ci-deploy-0-1--hail-is-hail.json
-     SPARK_VERSION=2.4.5
+     SPARK_VERSION=3.1.1
      BRANCH=0.2
      SHA="{{ code.sha }}"
      GS_JAR=gs://hail-common/builds/${BRANCH}/jars/hail-${BRANCH}-${SHA}-Spark-${SPARK_VERSION}.jar

ci/Dockerfile.test

@@ -4,4 +4,3 @@ COPY hail/python/setup-hailtop.py /hailtop/setup.py
 COPY hail/python/hailtop /hailtop/hailtop/
 RUN hail-pip-install /hailtop && rm -rf /hailtop
 COPY ci/test/ /test/
-RUN hail-pip-install pytest-instafail==0.4.1 pytest-asyncio==0.10.0

ci/test/resources/build.yaml

@@ -52,6 +52,36 @@ steps:
    publishAs: service-base
    dependsOn:
     - base_image
+ - kind: buildImage
+   name: create_certs_image
+   dockerFile: ci/test/resources/Dockerfile.certs
+   contextPath: ci/test/resources
+   publishAs: test_hello_create_certs_image
+   dependsOn:
+     - service_base_image
+ - kind: runImage
+   name: create_certs
+   image:
+     valueFrom: create_certs_image.image
+   script: |
+     set -ex
+     python3 create_certs.py \
+             {{ default_ns.name }} \
+             config.yaml \
+             /ssl-config-hail-root/hail-root-key.pem \
+             /ssl-config-hail-root/hail-root-cert.pem
+   serviceAccount:
+     name: admin
+     namespace:
+       valueFrom: default_ns.name
+   secrets:
+     - name: ssl-config-hail-root
+       namespace:
+         valueFrom: default_ns.name
+       mountPath: /ssl-config-hail-root
+   dependsOn:
+     - default_ns
+     - create_certs_image
  - kind: buildImage
    name: hello_image
    dockerFile: ci/test/resources/Dockerfile

ci/test/resources/config.yaml

@@ -0,0 +1,4 @@
+principals:
+- name: hello
+  domain: hello
+  kind: json