diff --git a/batch/batch/globals.py b/batch/batch/globals.py index c1a52e833cd..5014241eac1 100644 --- a/batch/batch/globals.py +++ b/batch/batch/globals.py @@ -21,7 +21,7 @@ BATCH_FORMAT_VERSION = 7 STATUS_FORMAT_VERSION = 5 -INSTANCE_VERSION = 25 +INSTANCE_VERSION = 26 MAX_PERSISTENT_SSD_SIZE_GIB = 64 * 1024 RESERVED_STORAGE_GB_PER_CORE = 5 diff --git a/batch/test/billing_projects.py b/batch/test/billing_projects.py index ef4f1ad9f45..991fc596ff9 100644 --- a/batch/test/billing_projects.py +++ b/batch/test/billing_projects.py @@ -10,7 +10,7 @@ def get_billing_project_prefix(): async def delete_all_test_billing_projects(): billing_project_prefix = get_billing_project_prefix() - bc = await BatchClient.create('', token_file=os.environ['HAIL_TEST_DEV_TOKEN_FILE']) + bc = await BatchClient.create('', cloud_credentials_file=os.environ['HAIL_TEST_DEV_GSA_KEY_FILE']) try: for project in await bc.list_billing_projects(): if project['billing_project'].startswith(billing_project_prefix): diff --git a/batch/test/test_accounts.py b/batch/test/test_accounts.py index 5a2673ae3d2..941f2c64b93 100644 --- a/batch/test/test_accounts.py +++ b/batch/test/test_accounts.py @@ -24,7 +24,7 @@ async def make_client() -> AsyncGenerator[Callable[[str], Awaitable[BatchClient] _bcs = [] async def factory(project: str): - bc = await BatchClient.create(project, token_file=os.environ['HAIL_TEST_TOKEN_FILE']) + bc = await BatchClient.create(project, cloud_credentials_file=os.environ['HAIL_TEST_GSA_KEY_FILE']) _bcs.append(bc) return bc @@ -36,7 +36,8 @@ async def factory(project: str): @pytest.fixture async def dev_client() -> AsyncGenerator[BatchClient, Any]: bc = await BatchClient.create( - 'billing-project-not-needed-but-required-by-BatchClient', token_file=os.environ['HAIL_TEST_DEV_TOKEN_FILE'] + 'billing-project-not-needed-but-required-by-BatchClient', + cloud_credentials_file=os.environ['HAIL_TEST_DEV_GSA_KEY_FILE'], ) yield bc await bc.close() diff --git a/build.yaml b/build.yaml index 43a42538360..d909a3b3ac2 100644 --- a/build.yaml +++ b/build.yaml @@ -1088,6 +1088,7 @@ steps: export HAIL_TEST_GCS_BUCKET={{ global.hail_test_gcs_bucket }} export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json export HAIL_TEST_S3_BUCKET=hail-test-dy5rg export AWS_SHARED_CREDENTIALS_FILE=/test-aws-key/credentials @@ -1213,6 +1214,7 @@ steps: # The test should use the test credentials, not CI's credentials sed -i 's/gsa-key/test-gsa-key/g' ${SPARK_HOME}/conf/core-site.xml export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json export HAIL_QUERY_N_CORES=2 export OMP_NUM_THREADS=2 @@ -1646,6 +1648,8 @@ steps: valueFrom: hailgenetics_hailtop_image.image script: | set -ex + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} {% if default_ns.name == "default" %} @@ -1658,10 +1662,10 @@ steps: {{ code.username }} {{ code.login_id }} {% endif %} secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key dependsOn: - default_ns - hailgenetics_hailtop_image @@ -1747,6 +1751,8 @@ steps: valueFrom: hail_dev_image.image script: | set -ex + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} python3 -m pytest \ @@ -1760,10 +1766,10 @@ steps: --timeout=120 \ /io/monitoring/test secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key timeout: 300 inputs: - from: /repo/monitoring/test @@ -1787,6 +1793,9 @@ steps: script: | set -ex + # Or else hailctl will try to authenticate as CI + unset HAIL_IDENTITY_PROVIDER_JSON + export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json @@ -1846,6 +1855,9 @@ steps: script: | set -ex + # Or else hailctl will try to authenticate as CI + unset HAIL_IDENTITY_PROVIDER_JSON + export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json @@ -2293,6 +2305,8 @@ steps: script: | set -ex + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} {% for user in code.get("developers", []) %} {% if user['username'] != 'test-dev' %} @@ -2304,10 +2318,10 @@ steps: {% endif %} {% endfor %} secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key scopes: - dev - test @@ -2417,10 +2431,6 @@ steps: namespace: valueFrom: default_ns.name mountPath: /test-gsa-key - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens dependsOn: - default_ns - merge_code @@ -2501,10 +2511,6 @@ steps: namespace: valueFrom: default_ns.name mountPath: /test-gsa-key - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens - name: auth-oauth2-client-secret namespace: valueFrom: default_ns.name @@ -2604,6 +2610,14 @@ steps: valueFrom: batch_image.image script: | set -ex + + # Use the test identity's credentials instead of CI's + export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + + export HAIL_TEST_GSA_KEY_FILE=/test-gsa-key/key.json + export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json + export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json export CI_UTILS_IMAGE={{ ci_utils_image.image }} export HAIL_CURL_IMAGE={{ curl_image.image }} @@ -2615,8 +2629,6 @@ steps: export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}" export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}" export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" - export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json - export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json export HAIL_TOKEN="{{ token }}" export HAIL_CLOUD="{{ global.cloud }}" export HAIL_DOMAIN="{{ global.domain }}" @@ -2645,18 +2657,14 @@ steps: port: 5000 timeout: 1500 secrets: - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens - - name: test-dev-tokens - namespace: - valueFrom: default_ns.name - mountPath: /dev-tokens - name: test-gsa-key namespace: valueFrom: default_ns.name mountPath: /test-gsa-key + - name: test-dev-gsa-key + namespace: + valueFrom: default_ns.name + mountPath: /test-dev-gsa-key dependsOn: - create_deploy_config - create_accounts @@ -2682,7 +2690,9 @@ steps: set -ex export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}" - export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export HAIL_TEST_DEV_GSA_KEY_FILE=/test-dev-gsa-key/key.json export HAIL_TOKEN="{{ test_batch.token }}" cd /io/test python3 -c ' @@ -2694,10 +2704,10 @@ steps: - from: /repo/batch/test to: /io/test secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /dev-tokens + mountPath: /test-dev-gsa-key alwaysRun: true dependsOn: - create_deploy_config @@ -2829,6 +2839,8 @@ steps: script: | set -ex export ORGANIZATION=hail-ci-test + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export REPO_NAME=ci-test-"{{ create_ci_test_repo.token }}" export NAMESPACE="{{ default_ns.name }}" @@ -2842,10 +2854,10 @@ steps: --durations=50 \ /io/ci/test secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key timeout: 5400 inputs: - from: /repo/ci/test @@ -2876,6 +2888,7 @@ steps: export HAIL_CLOUD={{ global.cloud }} export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}" export HAIL_GENETICS_HAIL_IMAGE="{{ hailgenetics_hail_image.image }}" export HAIL_GENETICS_HAILTOP_IMAGE="{{ hailgenetics_hailtop_image.image }}" @@ -2900,10 +2913,6 @@ steps: to: /io/hailtop timeout: 1200 secrets: - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens - name: test-gsa-key namespace: valueFrom: default_ns.name @@ -3050,6 +3059,7 @@ steps: set -ex export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json cd /io/hailtop/batch hailctl config set batch/billing_project test hailctl config set batch/remote_tmpdir {{ global.test_storage_uri }}/test_batch_docs/{{ token }}/ @@ -3067,10 +3077,6 @@ steps: --timeout=120 \ --ignore=docs/conf.py secrets: - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens - name: test-gsa-key namespace: valueFrom: default_ns.name @@ -3455,16 +3461,18 @@ steps: valueFrom: hailgenetics_hailtop_image.image script: | set -ex + export GOOGLE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-gsa-key/key.json hailctl curl {{ default_ns.name }} www / \ -vvv \ -fsSL \ --retry 3 \ --retry-delay 5 secrets: - - name: test-tokens + - name: test-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-gsa-key dependsOn: - default_ns - create_accounts @@ -3515,10 +3523,6 @@ steps: - from: /repo/hail/testng-fs.xml to: /io/testng-fs.xml secrets: - - name: test-tokens - namespace: - valueFrom: default_ns.name - mountPath: /user-tokens - name: test-gsa-key namespace: valueFrom: default_ns.name @@ -3586,6 +3590,8 @@ steps: valueFrom: hailgenetics_hailtop_image.image script: | export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json cat >cancel_all_running_test_batches.py <<'EOF' from hailtop.batch_client.aioclient import BatchClient @@ -3607,10 +3613,10 @@ steps: python3 cancel_all_running_test_batches.py secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key alwaysRun: true timeout: 300 dependsOn: @@ -3632,6 +3638,8 @@ steps: image: valueFrom: hail_dev_image.image script: | + export GOOGLE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json + export AZURE_APPLICATION_CREDENTIALS=/test-dev-gsa-key/key.json export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }} export DOCKER_PREFIX="{{ global.docker_prefix }}" export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}" @@ -3652,10 +3660,10 @@ steps: to: /io/test timeout: 300 secrets: - - name: test-dev-tokens + - name: test-dev-gsa-key namespace: valueFrom: default_ns.name - mountPath: /user-tokens + mountPath: /test-dev-gsa-key scopes: - test - dev diff --git a/hail/python/hailtop/auth/auth.py b/hail/python/hailtop/auth/auth.py index efd18937901..796bd9b206e 100644 --- a/hail/python/hailtop/auth/auth.py +++ b/hail/python/hailtop/auth/auth.py @@ -80,16 +80,17 @@ async def __aexit__(self, *_) -> None: def hail_credentials( *, tokens_file: Optional[str] = None, + cloud_credentials_file: Optional[str] = None, namespace: Optional[str] = None, authorize_target: bool = True ) -> HailCredentials: tokens = get_tokens(tokens_file) deploy_config = get_deploy_config() ns = namespace or deploy_config.default_namespace() - return HailCredentials(tokens, get_cloud_credentials_scoped_for_hail(), ns, authorize_target=authorize_target) + return HailCredentials(tokens, get_cloud_credentials_scoped_for_hail(credentials_file=cloud_credentials_file), ns, authorize_target=authorize_target) -def get_cloud_credentials_scoped_for_hail() -> Optional[CloudCredentials]: +def get_cloud_credentials_scoped_for_hail(credentials_file: Optional[str] = None) -> Optional[CloudCredentials]: scopes: Optional[List[str]] spec = load_identity_spec() @@ -100,6 +101,8 @@ def get_cloud_credentials_scoped_for_hail() -> Optional[CloudCredentials]: scopes = ['email', 'openid', 'profile'] if spec.oauth2_credentials is not None: return GoogleCredentials.from_credentials_data(spec.oauth2_credentials, scopes=scopes) + if credentials_file is not None: + return GoogleCredentials.from_file(credentials_file) return GoogleCredentials.default_credentials(scopes=scopes, anonymous_ok=False) assert spec.idp == IdentityProvider.MICROSOFT @@ -110,6 +113,9 @@ def get_cloud_credentials_scoped_for_hail() -> Optional[CloudCredentials]: scopes = [os.environ["HAIL_AZURE_OAUTH_SCOPE"]] else: scopes = None + + if credentials_file is not None: + return AzureCredentials.from_file(credentials_file, scopes=scopes) return AzureCredentials.default_credentials(scopes=scopes) diff --git a/hail/python/hailtop/batch_client/aioclient.py b/hail/python/hailtop/batch_client/aioclient.py index 922fdf93913..7e607c12074 100644 --- a/hail/python/hailtop/batch_client/aioclient.py +++ b/hail/python/hailtop/batch_client/aioclient.py @@ -850,7 +850,9 @@ async def create(billing_project: str, session: Optional[httpx.ClientSession] = None, headers: Optional[Dict[str, str]] = None, _token: Optional[str] = None, - token_file: Optional[str] = None): + token_file: Optional[str] = None, + *, + cloud_credentials_file: Optional[str] = None): if not deploy_config: deploy_config = get_deploy_config() url = deploy_config.base_url('batch') @@ -860,7 +862,7 @@ async def create(billing_project: str, if _token is not None: credentials = HailExplicitTokenCredentials(_token) else: - credentials = hail_credentials(tokens_file=token_file) + credentials = hail_credentials(tokens_file=token_file, cloud_credentials_file=cloud_credentials_file) return BatchClient( billing_project=billing_project, url=url,