From 54e3fad567a23966264c45bf914050e5a22fa581 Mon Sep 17 00:00:00 2001 From: Daniel Goldstein Date: Thu, 15 Jun 2023 15:23:41 -0400 Subject: [PATCH] [auth] More easily add developers to developer namespaces (#13180) I decided to break off this chunk from another PR that has stalled. That PR will ultimately build on this to add all developers automatically to dev AND test namespaces, but this should be an improvement for now. A few things in here: - Deleted all the `DatabaseResource` stuff in the auth driver. Since databases now are created and destroyed with the namespace and not the developer, this is basically dead code. - Added the ability to add a user for an existing hail identity. This is only permitted in dev namespaces and serves as a way for developers to use the same hail identity across namespaces. There is one caveat here: `create_initial_account.py` tries to copy the `-gsa-key` secret from default into the developer namespace and this code will *not* do that anymore. For the developer to submit jobs to the namespace, they must first manually copy in the secret from `default` if it does not already exist inside the namespace. This is awkward, but IMO acceptable because: - the copying code in `create_initial_account.py` is already broken anyway because when that script is run in a dev deploy it does not have access to production secrets - I hope that when we eventually go keyless we can delete the gsa key secrets and this whole problem goes away. - I feel like it's not too bad to do this manual one time copy as opposed to maintaining code that is privileged enough to reach across namespaces. Seems error prone and like a security headache. - Deletes `create_initial_account.py` in favor of using our actual API to create the dev user. --- auth/auth/auth.py | 65 +++++++++- auth/auth/driver/driver.py | 122 ++---------------- batch/batch/driver/job.py | 3 +- batch/batch/driver/main.py | 2 +- build.yaml | 58 +++++---- ci/bootstrap.py | 2 +- ci/create_initial_account.py | 99 -------------- gear/gear/__init__.py | 2 + .../batch/driver => gear/gear}/k8s_cache.py | 3 +- hail/python/hailtop/auth/auth.py | 17 ++- hail/python/hailtop/hailctl/auth/cli.py | 4 +- .../hailtop/hailctl/auth/create_user.py | 9 +- hail/python/hailtop/hailctl/dev/cli.py | 1 - 13 files changed, 126 insertions(+), 261 deletions(-) delete mode 100644 ci/create_initial_account.py rename {batch/batch/driver => gear/gear}/k8s_cache.py (95%) diff --git a/auth/auth/auth.py b/auth/auth/auth.py index 0ddf87686bb..a07229d7f32 100644 --- a/auth/auth/auth.py +++ b/auth/auth/auth.py @@ -7,6 +7,9 @@ import aiohttp import aiohttp_session +import kubernetes_asyncio.client +import kubernetes_asyncio.client.rest +import kubernetes_asyncio.config import uvloop from aiohttp import web from prometheus_async.aio.web import server_stats # type: ignore @@ -14,6 +17,7 @@ from gear import ( AuthClient, Database, + K8sCache, Transaction, check_csrf_token, create_session, @@ -53,6 +57,9 @@ CLOUD = get_global_config()['cloud'] ORGANIZATION_DOMAIN = os.environ['HAIL_ORGANIZATION_DOMAIN'] +DEFAULT_NAMESPACE = os.environ['HAIL_DEFAULT_NAMESPACE'] + +is_test_deployment = DEFAULT_NAMESPACE != 'default' deploy_config = get_deploy_config() @@ -124,7 +131,14 @@ async def check_valid_new_user(tx: Transaction, username, login_id, is_developer async def insert_new_user( - db: Database, username: str, login_id: Optional[str], is_developer: bool, is_service_account: bool + db: Database, + username: str, + login_id: Optional[str], + is_developer: bool, + is_service_account: bool, + *, + hail_identity: Optional[str] = None, + hail_credentials_secret_name: Optional[str] = None, ) -> bool: @transaction(db) async def _insert(tx): @@ -134,10 +148,18 @@ async def _insert(tx): await tx.execute_insertone( ''' -INSERT INTO users (state, username, login_id, is_developer, is_service_account) -VALUES (%s, %s, %s, %s, %s); +INSERT INTO users (state, username, login_id, is_developer, is_service_account, hail_identity, hail_credentials_secret_name) +VALUES (%s, %s, %s, %s, %s, %s, %s); ''', - ('creating', username, login_id, is_developer, is_service_account), + ( + 'creating', + username, + login_id, + is_developer, + is_service_account, + hail_identity, + hail_credentials_secret_name, + ), ) await _insert() # pylint: disable=no-value-for-parameter @@ -367,8 +389,29 @@ async def create_user(request: web.Request, userdata): # pylint: disable=unused is_developer = body['is_developer'] is_service_account = body['is_service_account'] + hail_identity = body.get('hail_identity') + hail_credentials_secret_name = body.get('hail_credentials_secret_name') + if (hail_identity or hail_credentials_secret_name) and not is_test_deployment: + raise web.HTTPBadRequest(text='Cannot specify an existing hail identity for a new user') + if hail_credentials_secret_name: + try: + k8s_cache: K8sCache = request.app['k8s_cache'] + await k8s_cache.read_secret(hail_credentials_secret_name, DEFAULT_NAMESPACE) + except kubernetes_asyncio.client.rest.ApiException as e: + raise web.HTTPBadRequest( + text=f'hail credentials secret name specified but was not found in namespace {DEFAULT_NAMESPACE}: {hail_credentials_secret_name}' + ) from e + try: - await insert_new_user(db, username, login_id, is_developer, is_service_account) + await insert_new_user( + db, + username, + login_id, + is_developer, + is_service_account, + hail_identity=hail_identity, + hail_credentials_secret_name=hail_credentials_secret_name, + ) except AuthUserError as e: raise e.http_response() @@ -750,12 +793,20 @@ async def on_startup(app): app['client_session'] = httpx.client_session() app['flow_client'] = get_flow_client('/auth-oauth2-client-secret/client_secret.json') + kubernetes_asyncio.config.load_incluster_config() + app['k8s_client'] = kubernetes_asyncio.client.CoreV1Api() + app['k8s_cache'] = K8sCache(app['k8s_client']) + async def on_cleanup(app): try: - await app['db'].async_close() + k8s_client: kubernetes_asyncio.client.CoreV1Api = app['k8s_client'] + await k8s_client.api_client.rest_client.pool_manager.close() finally: - await app['client_session'].close() + try: + await app['db'].async_close() + finally: + await app['client_session'].close() class AuthAccessLogger(AccessLogger): diff --git a/auth/auth/driver/driver.py b/auth/auth/driver/driver.py index 598155feb6b..94caac920d4 100644 --- a/auth/auth/driver/driver.py +++ b/auth/auth/driver/driver.py @@ -4,8 +4,7 @@ import logging import os import random -import secrets -from typing import Any, Awaitable, Callable, Dict, List, Optional +from typing import Any, Awaitable, Callable, Dict, List import aiohttp import kubernetes_asyncio.client @@ -17,7 +16,6 @@ from gear.cloud_config import get_gcp_config, get_global_config from hailtop import aiotools, httpx from hailtop import batch_client as bc -from hailtop.auth.sql_config import SQLConfig, create_secret_data_from_config from hailtop.utils import secret_alnum_string, time_msecs log = logging.getLogger('auth.driver') @@ -34,7 +32,7 @@ class DatabaseConflictError(Exception): class EventHandler: - def __init__(self, handler, event=None, bump_secs=60.0, min_delay_secs=0.1): + def __init__(self, handler, event=None, bump_secs=5.0, min_delay_secs=0.1): self.handler = handler if event is None: event = asyncio.Event() @@ -234,86 +232,6 @@ async def delete(self): self.app_obj_id = None -class DatabaseResource: - def __init__(self, db_instance, name=None): - self.db_instance = db_instance - self.name = name - self.password = None - - async def create(self, name): - assert self.name is None - - if is_test_deployment: - return - - await self._delete(name) - - self.password = secrets.token_urlsafe(16) - await self.db_instance.just_execute( - f''' -CREATE DATABASE `{name}`; - -CREATE USER '{name}'@'%' IDENTIFIED BY '{self.password}'; -GRANT ALL ON `{name}`.* TO '{name}'@'%'; -''' - ) - self.name = name - - def secret_data(self): - with open('/database-server-config/sql-config.json', 'r', encoding='utf-8') as f: - server_config = SQLConfig.from_json(f.read()) - with open('/database-server-config/server-ca.pem', 'r', encoding='utf-8') as f: - server_ca = f.read() - client_cert: Optional[str] - client_key: Optional[str] - if server_config.using_mtls(): - with open('/database-server-config/client-cert.pem', 'r', encoding='utf-8') as f: - client_cert = f.read() - with open('/database-server-config/client-key.pem', 'r', encoding='utf-8') as f: - client_key = f.read() - else: - client_cert = None - client_key = None - - if is_test_deployment: - return create_secret_data_from_config(server_config, server_ca, client_cert, client_key) - - assert self.name is not None - assert self.password is not None - - config = SQLConfig( - host=server_config.host, - port=server_config.port, - user=self.name, - password=self.password, - instance=server_config.instance, - connection_name=server_config.connection_name, - db=self.name, - ssl_ca='/sql-config/server-ca.pem', - ssl_cert='/sql-config/client-cert.pem' if client_cert is not None else None, - ssl_key='/sql-config/client-key.pem' if client_key is not None else None, - ssl_mode='VERIFY_CA', - ) - return create_secret_data_from_config(config, server_ca, client_cert, client_key) - - async def _delete(self, name): - if is_test_deployment: - return - - # no DROP USER IF EXISTS in current db version - row = await self.db_instance.execute_and_fetchone('SELECT 1 FROM mysql.user WHERE User = %s;', (name,)) - if row is not None: - await self.db_instance.just_execute(f"DROP USER '{name}';") - - await self.db_instance.just_execute(f'DROP DATABASE IF EXISTS `{name}`;') - - async def delete(self): - if self.name is None: - return - await self._delete(self.name) - self.name = None - - class K8sNamespaceResource: def __init__(self, k8s_client, name=None): self.k8s_client = k8s_client @@ -410,7 +328,6 @@ async def delete(self): async def _create_user(app, user, skip_trial_bp, cleanup): - db_instance = app['db_instance'] db = app['db'] k8s_client = app['k8s_client'] identity_client = app['identity_client'] @@ -481,21 +398,14 @@ async def _create_user(app, user, skip_trial_bp, cleanup): updates['hail_credentials_secret_name'] = hail_credentials_secret_name namespace_name = user['namespace_name'] - if namespace_name is None and user['is_developer'] == 1: + # auth services in test namespaces cannot/should not be creating and deleting namespaces + if namespace_name is None and user['is_developer'] == 1 and not is_test_deployment: namespace_name = ident namespace = K8sNamespaceResource(k8s_client) cleanup.append(namespace.delete) await namespace.create(namespace_name) updates['namespace_name'] = namespace_name - db_resource = DatabaseResource(db_instance) - cleanup.append(db_resource.delete) - await db_resource.create(ident) - - db_secret = K8sSecretResource(k8s_client) - cleanup.append(db_secret.delete) - await db_secret.create('database-server-config', namespace_name, db_resource.secret_data()) - if not skip_trial_bp and user['is_service_account'] != 1: trial_bp = user['trial_bp_name'] if trial_bp is None: @@ -536,7 +446,6 @@ async def create_user(app, user, skip_trial_bp=False): async def delete_user(app, user): - db_instance = app['db_instance'] db = app['db'] k8s_client = app['k8s_client'] identity_client = app['identity_client'] @@ -572,9 +481,6 @@ async def delete_user(app, user): namespace = K8sNamespaceResource(k8s_client, namespace_name) await namespace.delete() - db_resource = DatabaseResource(db_instance, user['username']) - await db_resource.delete() - trial_bp_name = user['trial_bp_name'] if trial_bp_name is not None: batch_client = app['batch_client'] @@ -619,10 +525,6 @@ async def async_main(): app['client_session'] = httpx.client_session() - db_instance = Database() - await db_instance.async_init(maxsize=50, config_file='/database-server-config/sql-config.json') - app['db_instance'] = db_instance - kubernetes_asyncio.config.load_incluster_config() app['k8s_client'] = kubernetes_asyncio.client.CoreV1Api() @@ -647,18 +549,14 @@ async def users_changed_handler(): await app['db'].async_close() finally: try: - if 'db_instance_pool' in app: - await app['db_instance_pool'].async_close() + await app['client_session'].close() finally: try: - await app['client_session'].close() + if user_creation_loop is not None: + user_creation_loop.shutdown() finally: try: - if user_creation_loop is not None: - user_creation_loop.shutdown() + await app['identity_client'].close() finally: - try: - await app['identity_client'].close() - finally: - k8s_client: kubernetes_asyncio.client.CoreV1Api = app['k8s_client'] - await k8s_client.api_client.rest_client.pool_manager.close() + k8s_client: kubernetes_asyncio.client.CoreV1Api = app['k8s_client'] + await k8s_client.api_client.rest_client.pool_manager.close() diff --git a/batch/batch/driver/job.py b/batch/batch/driver/job.py index d7b1ce876a5..411a802f730 100644 --- a/batch/batch/driver/job.py +++ b/batch/batch/driver/job.py @@ -8,7 +8,7 @@ import aiohttp -from gear import Database +from gear import Database, K8sCache from hailtop import httpx from hailtop.aiotools import BackgroundTaskManager from hailtop.utils import Notice, retry_transient_errors, time_msecs @@ -21,7 +21,6 @@ from ..instance_config import QuantifiedResource from ..spec_writer import SpecWriter from .instance import Instance -from .k8s_cache import K8sCache if TYPE_CHECKING: from .instance_collection import InstanceCollectionManager # pylint: disable=cyclic-import diff --git a/batch/batch/driver/main.py b/batch/batch/driver/main.py index 403cc82b6c3..f9c86a27d90 100644 --- a/batch/batch/driver/main.py +++ b/batch/batch/driver/main.py @@ -25,6 +25,7 @@ from gear import ( AuthClient, Database, + K8sCache, check_csrf_token, json_request, json_response, @@ -59,7 +60,6 @@ from .driver import CloudDriver from .instance_collection import InstanceCollectionManager, JobPrivateInstanceManager, Pool from .job import mark_job_complete, mark_job_started -from .k8s_cache import K8sCache uvloop.install() diff --git a/build.yaml b/build.yaml index 3a3b83c0b2a..df96435e711 100644 --- a/build.yaml +++ b/build.yaml @@ -576,34 +576,6 @@ steps: - deploy_auth_driver_service_account - create_test_gsa_keys - create_test_database_server_config - - kind: runImage - name: create_initial_user - runIfRequested: true - image: - valueFrom: auth_image.image - script: | - set -ex - export NAMESPACE={{ default_ns.name }} - export CLOUD={{ global.cloud }} - python3 /io/create_initial_account.py {{ code.username }} {{ code.login_id }} - serviceAccount: - name: admin - namespace: - valueFrom: default_ns.name - secrets: - - name: - valueFrom: auth_database.user_secret_name - namespace: - valueFrom: default_ns.name - mountPath: /sql-config - inputs: - - from: /repo/ci/create_initial_account.py - to: /io/create_initial_account.py - dependsOn: - - default_ns - - auth_image - - merge_code - - auth_database - kind: buildImage2 name: hailgenetics_vep_grch37_85_image dockerFile: /io/repo/docker/hailgenetics/vep/grch37/85/Dockerfile @@ -1725,6 +1697,36 @@ steps: - create_dummy_oauth2_client_secret - create_certs - create_accounts + - kind: runImage + name: create_initial_user + runIfRequested: true + image: + valueFrom: hailgenetics_hailtop_image.image + script: | + set -ex + {% if default_ns.name == "default" %} + hailctl auth create-user --developer {{ code.username }} {{ code.login_id }} + {% else %} + hailctl auth create-user \ + --developer \ + --hail-identity {{ code.hail_identity }} \ + --hail-credentials-secret-name {{ code.username }}-gsa-key \ + {{ code.username }} {{ code.login_id }} + {% endif %} + secrets: + - name: worker-deploy-config + namespace: + valueFrom: default_ns.name + mountPath: /deploy-config + - name: test-dev-tokens + namespace: + valueFrom: default_ns.name + mountPath: /user-tokens + dependsOn: + - default_ns + - hailgenetics_hailtop_image + - merge_code + - deploy_auth - kind: runImage name: delete_monitoring_tables image: diff --git a/ci/bootstrap.py b/ci/bootstrap.py index 894eb0c8011..839dec64fd2 100644 --- a/ci/bootstrap.py +++ b/ci/bootstrap.py @@ -9,11 +9,11 @@ import kubernetes_asyncio.client import kubernetes_asyncio.config -from batch.driver.k8s_cache import K8sCache from ci.build import BuildConfiguration, Code from ci.environment import KUBERNETES_SERVER_URL, STORAGE_URI from ci.github import clone_or_fetch_script from ci.utils import generate_token +from gear import K8sCache from hailtop.utils import check_shell_output BATCH_WORKER_IMAGE = os.environ['BATCH_WORKER_IMAGE'] diff --git a/ci/create_initial_account.py b/ci/create_initial_account.py deleted file mode 100644 index 46d7950dc25..00000000000 --- a/ci/create_initial_account.py +++ /dev/null @@ -1,99 +0,0 @@ -import argparse -import base64 -import json -import os - -import kubernetes_asyncio.client -import kubernetes_asyncio.config - -from gear import Database, transaction -from hailtop.utils import async_to_blocking - -NAMESPACE = os.environ['NAMESPACE'] - - -async def copy_identity_from_default(hail_credentials_secret_name: str) -> str: - cloud = os.environ['CLOUD'] - await kubernetes_asyncio.config.load_kube_config() - k8s_client = kubernetes_asyncio.client.CoreV1Api() - - secret = await k8s_client.read_namespaced_secret(hail_credentials_secret_name, 'default') - - try: - await k8s_client.delete_namespaced_secret(hail_credentials_secret_name, NAMESPACE) - except kubernetes_asyncio.client.rest.ApiException as e: - if e.status == 404: - pass - else: - raise - - await k8s_client.create_namespaced_secret( - NAMESPACE, - kubernetes_asyncio.client.V1Secret( - metadata=kubernetes_asyncio.client.V1ObjectMeta(name=hail_credentials_secret_name), - data=secret.data, - ), - ) - - credentials_json = base64.b64decode(secret.data['key.json']).decode() - credentials = json.loads(credentials_json) - - if cloud == 'gcp': - return credentials['client_email'] - assert cloud == 'azure' - return credentials['appObjectId'] - - -async def insert_user_if_not_exists(db, username, login_id, is_developer, is_service_account): - @transaction(db) - async def insert(tx): - row = await tx.execute_and_fetchone('SELECT id, state FROM users where username = %s;', (username,)) - if row: - if row['state'] == 'active': - return None - return row['id'] - - if NAMESPACE == 'default': - hail_credentials_secret_name = None - hail_identity = None - namespace_name = None - else: - hail_credentials_secret_name = f'{username}-gsa-key' - hail_identity = await copy_identity_from_default(hail_credentials_secret_name) - namespace_name = NAMESPACE - - return await tx.execute_insertone( - ''' - INSERT INTO users (state, username, login_id, is_developer, is_service_account, hail_identity, hail_credentials_secret_name, namespace_name) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s); - ''', - ( - 'creating', - username, - login_id, - is_developer, - is_service_account, - hail_identity, - hail_credentials_secret_name, - namespace_name, - ), - ) - - return await insert() # pylint: disable=no-value-for-parameter - - -async def main(): - parser = argparse.ArgumentParser(description='Create an initial dev user.') - - parser.add_argument('username', help='The username of the initial user.') - parser.add_argument('login_id', metavar='login-id', help='The login id of the initial user.') - - args = parser.parse_args() - - db = Database() - await db.async_init(maxsize=50) - - await insert_user_if_not_exists(db, args.username, args.login_id, True, False) - - -async_to_blocking(main()) diff --git a/gear/gear/__init__.py b/gear/gear/__init__.py index 554560e31eb..a323f65fc7a 100644 --- a/gear/gear/__init__.py +++ b/gear/gear/__init__.py @@ -3,6 +3,7 @@ from .csrf import check_csrf_token, new_csrf_token from .database import Database, Transaction, create_database_pool, resolve_test_db_endpoint, transaction from .http_server_utils import json_request, json_response +from .k8s_cache import K8sCache from .metrics import monitor_endpoints_middleware from .session import setup_aiohttp_session @@ -22,4 +23,5 @@ 'json_request', 'json_response', 'resolve_test_db_endpoint', + 'K8sCache', ] diff --git a/batch/batch/driver/k8s_cache.py b/gear/gear/k8s_cache.py similarity index 95% rename from batch/batch/driver/k8s_cache.py rename to gear/gear/k8s_cache.py index cdd14528fd6..c53f31e56a2 100644 --- a/batch/batch/driver/k8s_cache.py +++ b/gear/gear/k8s_cache.py @@ -1,9 +1,10 @@ import os from typing import Tuple -from gear.time_limited_max_size_cache import TimeLimitedMaxSizeCache from hailtop.utils import retry_transient_errors +from .time_limited_max_size_cache import TimeLimitedMaxSizeCache + FIVE_SECONDS_NS = 5 * 1000 * 1000 * 1000 diff --git a/hail/python/hailtop/auth/auth.py b/hail/python/hailtop/auth/auth.py index aef8de93133..6da77af4504 100644 --- a/hail/python/hailtop/auth/auth.py +++ b/hail/python/hailtop/auth/auth.py @@ -152,17 +152,24 @@ async def async_get_user(username: str, namespace: Optional[str] = None) -> dict ) -def create_user(username: str, login_id: str, is_developer: bool, is_service_account: bool, namespace: Optional[str] = None): - return async_to_blocking(async_create_user(username, login_id, is_developer, is_service_account, namespace=namespace)) - - -async def async_create_user(username: str, login_id: str, is_developer: bool, is_service_account: bool, namespace: Optional[str] = None): +async def async_create_user( + username: str, + login_id: str, + is_developer: bool, + is_service_account: bool, + hail_identity: Optional[str], + hail_credentials_secret_name: Optional[str], + *, + namespace: Optional[str] = None +): deploy_config, headers, _ = deploy_config_and_headers_from_namespace(namespace) body = { 'login_id': login_id, 'is_developer': is_developer, 'is_service_account': is_service_account, + 'hail_identity': hail_identity, + 'hail_credentials_secret_name': hail_credentials_secret_name, } async with httpx.client_session( diff --git a/hail/python/hailtop/hailctl/auth/cli.py b/hail/python/hailtop/hailctl/auth/cli.py index 9ee268b9019..65584200c52 100644 --- a/hail/python/hailtop/hailctl/auth/cli.py +++ b/hail/python/hailtop/hailctl/auth/cli.py @@ -91,6 +91,8 @@ def create_user( login_id: Ann[str, Arg(help="In Azure, the user's object ID in AAD. In GCP, the Google email")], developer: bool = False, service_account: bool = False, + hail_identity: Optional[str] = None, + hail_credentials_secret_name: Optional[str] = None, namespace: NamespaceOption = None, wait: bool = False, ): @@ -99,7 +101,7 @@ def create_user( ''' from .create_user import polling_create_user # pylint: disable=import-outside-toplevel - asyncio.run(polling_create_user(username, login_id, developer, service_account, namespace, wait)) + asyncio.run(polling_create_user(username, login_id, developer, service_account, hail_identity, hail_credentials_secret_name, namespace=namespace, wait=wait)) @app.command() diff --git a/hail/python/hailtop/hailctl/auth/create_user.py b/hail/python/hailtop/hailctl/auth/create_user.py index da80b953615..061aa8495f6 100644 --- a/hail/python/hailtop/hailctl/auth/create_user.py +++ b/hail/python/hailtop/hailctl/auth/create_user.py @@ -13,11 +13,14 @@ async def polling_create_user( login_id: str, developer: bool, service_account: bool, - namespace: Optional[str], - wait: bool, + hail_identity: Optional[str], + hail_credentials_secret_name: Optional[str], + *, + namespace: Optional[str] = None, + wait: bool = False, ): try: - await async_create_user(username, login_id, developer, service_account, namespace) + await async_create_user(username, login_id, developer, service_account, hail_identity, hail_credentials_secret_name, namespace=namespace) if not wait: return diff --git a/hail/python/hailtop/hailctl/dev/cli.py b/hail/python/hailtop/hailctl/dev/cli.py index 23d40744c70..23b6ed98879 100644 --- a/hail/python/hailtop/hailctl/dev/cli.py +++ b/hail/python/hailtop/hailctl/dev/cli.py @@ -53,7 +53,6 @@ async def _deploy(branch: str, steps: List[str], excluded_steps: List[str], extr deploy_config = get_deploy_config() steps = unpack_comma_delimited_inputs(steps) - print(steps) excluded_steps = unpack_comma_delimited_inputs(excluded_steps) extra_config_dict = unpack_key_value_inputs(extra_config) async with CIClient(deploy_config) as ci_client: