diff --git a/build.yaml b/build.yaml index 9c442d02b43..5a402458209 100644 --- a/build.yaml +++ b/build.yaml @@ -255,35 +255,43 @@ steps: dependsOn: - base_image - merge_code - - kind: createDatabase - name: test_database_instance - databaseName: test-instance - image: - valueFrom: ci_utils_image.image - migrations: [] - namespace: - valueFrom: default_ns.name - scopes: - - test - dependsOn: - - default_ns - - ci_utils_image - kind: runImage - name: create_database_server_config + name: create_test_database_server_config image: - valueFrom: ci_utils_image.image + valueFrom: create_certs_image.image script: | - kubectl -n {{ default_ns.name }} get -o json secret {{ test_database_instance.admin_secret_name }} | jq '{apiVersion, kind, type, data, metadata: {name: "database-server-config"}}' | kubectl -n {{ default_ns.name }} apply -f - + set -ex + + if ! kubectl get secret -n {{ default_ns.name }} database-server-config; + then + NAMESPACE={{ default_ns.name }} bash /create_test_db_config.sh + fi serviceAccount: name: admin namespace: valueFrom: default_ns.name scopes: + - dev - test dependsOn: - default_ns - - ci_utils_image - - test_database_instance + - create_certs_image + - kind: deploy + name: deploy_test_db + namespace: + valueFrom: default_ns.name + config: docker/mysql/db.yaml + wait: + - kind: Service + name: db + for: alive + resource_type: statefulset + scopes: + - dev + - test + dependsOn: + - default_ns + - create_test_database_server_config - kind: buildImage2 name: admin_pod_image dockerFile: /io/repo/admin-pod/Dockerfile @@ -316,8 +324,8 @@ steps: - default_ns - admin_pod_image - merge_code - - create_database_server_config - - kind: createDatabase + - create_test_database_server_config + - kind: createDatabase2 name: auth_database databaseName: auth image: @@ -354,6 +362,8 @@ steps: - merge_code - delete_auth_tables - ci_utils_image + - create_test_database_server_config + - deploy_test_db - kind: runImage name: create_deploy_config image: @@ -525,7 +535,7 @@ steps: - create_deploy_config - deploy_auth_driver_service_account - create_test_gsa_keys - - create_database_server_config + - create_test_database_server_config - kind: runImage name: create_initial_user runIfRequested: true @@ -1697,8 +1707,8 @@ steps: - default_ns - admin_pod_image - merge_code - - create_database_server_config - - kind: createDatabase + - create_test_database_server_config + - kind: createDatabase2 name: monitoring_database databaseName: monitoring image: @@ -1721,6 +1731,8 @@ steps: - merge_code - delete_monitoring_tables - ci_utils_image + - create_test_database_server_config + - deploy_test_db - kind: deploy name: deploy_monitoring namespace: @@ -1908,7 +1920,7 @@ steps: - default_ns - admin_pod_image - merge_code - - create_database_server_config + - create_test_database_server_config - kind: runImage name: delete_ci_tables image: @@ -1931,8 +1943,8 @@ steps: - default_ns - admin_pod_image - merge_code - - create_database_server_config - - kind: createDatabase + - create_test_database_server_config + - kind: createDatabase2 name: ci_database databaseName: ci image: @@ -1966,7 +1978,9 @@ steps: - merge_code - delete_ci_tables - ci_utils_image - - kind: createDatabase + - create_test_database_server_config + - deploy_test_db + - kind: createDatabase2 name: batch_database databaseName: batch image: @@ -2200,6 +2214,8 @@ steps: - merge_code - delete_batch_tables - ci_utils_image + - create_test_database_server_config + - deploy_test_db - kind: deploy name: deploy_batch namespace: @@ -2663,6 +2679,7 @@ steps: cp -R /io/repo/ci/* ./ci/ cp /io/repo/tls/Dockerfile ./ci/test/resources/Dockerfile.certs cp /io/repo/tls/create_certs.py ./ci/test/resources/ + cp /io/repo/tls/create_test_db_config.sh ./ci/test/resources/ cp /io/repo/pylintrc ./ cp /io/repo/setup.cfg ./ cp /io/repo/pyproject.toml ./ @@ -2727,7 +2744,6 @@ steps: for: alive dependsOn: - default_ns - - create_database_server_config - ci_image - ci_utils_image - create_accounts @@ -3007,7 +3023,7 @@ steps: inputs: - from: /repo/hail/python/hailtop to: /io/hailtop - - kind: createDatabase + - kind: createDatabase2 name: notebook_database databaseName: notebook image: @@ -3029,6 +3045,8 @@ steps: - default_ns - merge_code - ci_utils_image + - create_test_database_server_config + - deploy_test_db - kind: deploy name: deploy_notebook namespace: diff --git a/ci/ci/build.py b/ci/ci/build.py index 4f2eb0534d0..0752075a6a6 100644 --- a/ci/ci/build.py +++ b/ci/ci/build.py @@ -1190,14 +1190,10 @@ def __init__(self, params, database_name, namespace, migrations, shutdowns, inpu self.create_database_job = None self.cleanup_job = None - self.cant_create_database = is_test_deployment + self.cant_create_database = False # MySQL user name can be up to 16 characters long before MySQL 5.7.8 (32 after) - if self.cant_create_database: - self._name = None - self.admin_username = None - self.user_username = None - elif params.scope == 'deploy': + if params.scope == 'deploy': self._name = database_name self.admin_username = f'{database_name}-admin' self.user_username = f'{database_name}-user' diff --git a/ci/create_database.py b/ci/create_database.py index 036d5ec7180..6aac34c80d3 100644 --- a/ci/create_database.py +++ b/ci/create_database.py @@ -8,7 +8,9 @@ from shlex import quote as shq from typing import Optional -from gear import Database +import orjson + +from gear import Database, resolve_test_db_endpoint from hailtop.auth.sql_config import SQLConfig, create_secret_data_from_config from hailtop.utils import check_shell, check_shell_output @@ -62,19 +64,8 @@ async def create_database(): namespace = create_database_config['namespace'] database_name = create_database_config['database_name'] - cant_create_database = create_database_config['cant_create_database'] - - if cant_create_database: - assert sql_config.db is not None - - await write_user_config(namespace, database_name, 'admin', sql_config) - await write_user_config(namespace, database_name, 'user', sql_config) - return - scope = create_database_config['scope'] _name = create_database_config['_name'] - admin_username = create_database_config['admin_username'] - user_username = create_database_config['user_username'] db = Database() await db.async_init() @@ -89,79 +80,54 @@ async def create_database(): assert len(rows) == 1 return - with open(create_database_config['admin_password_file'], encoding='utf-8') as f: - admin_password = f.read() - - with open(create_database_config['user_password_file'], encoding='utf-8') as f: - user_password = f.read() - - admin_exists = await db.execute_and_fetchone("SELECT user FROM mysql.user WHERE user=%s", (admin_username,)) - admin_exists = admin_exists and admin_exists.get('user') == admin_username - - user_exists = await db.execute_and_fetchone("SELECT user FROM mysql.user WHERE user=%s", (user_username,)) - user_exists = user_exists and user_exists.get('user') == user_username - - create_admin_or_alter_password = ( - f"CREATE USER '{admin_username}'@'%' IDENTIFIED BY '{admin_password}';" - if not admin_exists - else "ALTER USER '{admin_username}'@'%' IDENTIFIED BY '{admin_password}';" - ) + async def create_user_if_doesnt_exist(admin_or_user, mysql_username, mysql_password): + existing_user = await db.execute_and_fetchone('SELECT 1 FROM mysql.user WHERE user=%s', (mysql_username,)) + if existing_user is not None: + return - create_user_or_alter_password = ( - f"CREATE USER '{user_username}'@'%' IDENTIFIED BY '{user_password}';" - if not user_exists - else "ALTER USER '{user_username}'@'%' IDENTIFIED BY '{user_password}';" - ) + if admin_or_user == 'admin': + allowed_operations = 'ALL' + else: + assert admin_or_user == 'user' + allowed_operations = 'SELECT, INSERT, UPDATE, DELETE, EXECUTE' - await db.just_execute( - f''' - CREATE DATABASE IF NOT EXISTS `{_name}`; + await db.just_execute( + f''' + CREATE USER '{mysql_username}'@'%' IDENTIFIED BY '{mysql_password}'; + GRANT {allowed_operations} ON `{_name}`.* TO '{mysql_username}'@'%'; + ''' + ) - {create_admin_or_alter_password} - GRANT ALL ON `{_name}`.* TO '{admin_username}'@'%'; + await write_user_config( + namespace, + database_name, + admin_or_user, + SQLConfig( + host=sql_config.host, + port=sql_config.port, + instance=sql_config.instance, + connection_name=sql_config.connection_name, + user=mysql_username, + password=mysql_password, + db=_name, + ssl_ca=sql_config.ssl_ca, + ssl_cert=sql_config.ssl_cert, + ssl_key=sql_config.ssl_key, + ssl_mode=sql_config.ssl_mode, + ), + ) - {create_user_or_alter_password} - GRANT SELECT, INSERT, UPDATE, DELETE, EXECUTE ON `{_name}`.* TO '{user_username}'@'%'; - ''' - ) + admin_username = create_database_config['admin_username'] + user_username = create_database_config['user_username'] - await write_user_config( - namespace, - database_name, - 'admin', - SQLConfig( - host=sql_config.host, - port=sql_config.port, - instance=sql_config.instance, - connection_name=sql_config.connection_name, - user=admin_username, - password=admin_password, - db=_name, - ssl_ca=sql_config.ssl_ca, - ssl_cert=sql_config.ssl_cert, - ssl_key=sql_config.ssl_key, - ssl_mode=sql_config.ssl_mode, - ), - ) + with open(create_database_config['admin_password_file'], encoding='utf-8') as f: + admin_password = f.read() + with open(create_database_config['user_password_file'], encoding='utf-8') as f: + user_password = f.read() - await write_user_config( - namespace, - database_name, - 'user', - SQLConfig( - host=sql_config.host, - port=sql_config.port, - instance=sql_config.instance, - connection_name=sql_config.connection_name, - user=user_username, - password=user_password, - db=_name, - ssl_ca=sql_config.ssl_ca, - ssl_cert=sql_config.ssl_cert, - ssl_key=sql_config.ssl_key, - ssl_mode=sql_config.ssl_mode, - ), - ) + await db.just_execute(f'CREATE DATABASE IF NOT EXISTS `{_name}`') + await create_user_if_doesnt_exist('admin', admin_username, admin_password) + await create_user_if_doesnt_exist('user', user_username, user_password) did_shutdown = False @@ -256,11 +222,15 @@ async def async_main(): ) admin_secret = json.loads(out) + admin_sql_config = SQLConfig.from_json(base64.b64decode(admin_secret['data']['sql-config.json']).decode()) + if namespace != 'default' and admin_sql_config.host.endswith('.svc.cluster.local'): + admin_sql_config = await resolve_test_db_endpoint(admin_sql_config) + with open('/sql-config.json', 'wb') as f: - f.write(base64.b64decode(admin_secret['data']['sql-config.json'])) + f.write(orjson.dumps(admin_sql_config.to_dict())) with open('/sql-config.cnf', 'wb') as f: - f.write(base64.b64decode(admin_secret['data']['sql-config.cnf'])) + f.write(admin_sql_config.to_cnf().encode('utf-8')) os.environ['HAIL_DATABASE_CONFIG_FILE'] = '/sql-config.json' os.environ['HAIL_SCOPE'] = scope diff --git a/ci/create_initial_account.py b/ci/create_initial_account.py index bca69de06de..46d7950dc25 100644 --- a/ci/create_initial_account.py +++ b/ci/create_initial_account.py @@ -19,6 +19,14 @@ async def copy_identity_from_default(hail_credentials_secret_name: str) -> str: secret = await k8s_client.read_namespaced_secret(hail_credentials_secret_name, 'default') + try: + await k8s_client.delete_namespaced_secret(hail_credentials_secret_name, NAMESPACE) + except kubernetes_asyncio.client.rest.ApiException as e: + if e.status == 404: + pass + else: + raise + await k8s_client.create_namespaced_secret( NAMESPACE, kubernetes_asyncio.client.V1Secret( diff --git a/ci/test/resources/build.yaml b/ci/test/resources/build.yaml index 662c873590f..d3b89a2b81a 100644 --- a/ci/test/resources/build.yaml +++ b/ci/test/resources/build.yaml @@ -236,6 +236,10 @@ steps: namespace: valueFrom: default_ns.name mountPath: /sql-config + serviceAccount: + name: admin + namespace: + valueFrom: default_ns.name dependsOn: - default_ns - hello_database diff --git a/docker/mysql/db.yaml b/docker/mysql/db.yaml new file mode 100644 index 00000000000..cab0f39f2d8 --- /dev/null +++ b/docker/mysql/db.yaml @@ -0,0 +1,104 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-server-conf +data: + my.cnf: | + [mysqld] + ssl_ca=/sql-config/server-ca.pem + ssl_cert=/sql-config/server-cert.pem + ssl_key=/sql-config/server-key.pem + require_secure_transport=ON + log_bin_trust_function_creators=ON + max_connections=100 + bind-address=0.0.0.0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: db + labels: + app: db +spec: + selector: + matchLabels: + app: db + serviceName: db + replicas: 1 + template: + metadata: + labels: + app: db + spec: + nodeSelector: + preemptible: "false" + containers: + - name: db + image: mysql:8.0.28 + livenessProbe: + exec: + command: ["mysqladmin", "ping"] + initialDelaySeconds: 30 + periodSeconds: 5 + resources: + requests: + cpu: "1.5" + memory: "1G" + limits: + cpu: "2" + memory: "1.5G" + env: + - name: MYSQL_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: database-server-config + key: db-root-password + - name: MYSQL_ROOT_HOST + value: "%" + volumeMounts: + - name: mysql-persistent-storage + mountPath: /var/lib/mysql + - name: database-server-config + mountPath: /sql-config + - name: mysql-conf + mountPath: /etc/mysql/conf.d + - name: mysql-client-conf + mountPath: /root + volumes: + - name: mysql-persistent-storage + persistentVolumeClaim: + claimName: mysql-pv-claim + - name: database-server-config + secret: + secretName: database-server-config + - name: mysql-client-conf + secret: + secretName: database-server-config + items: + - key: sql-config.cnf + path: .my.cnf + - name: mysql-conf + configMap: + name: mysql-server-conf + volumeClaimTemplates: + - metadata: + name: mysql-persistent-storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: db +spec: + type: NodePort + ports: + - protocol: TCP + port: 3306 + targetPort: 3306 + selector: + app: db diff --git a/gear/gear/__init__.py b/gear/gear/__init__.py index 339fb558118..554560e31eb 100644 --- a/gear/gear/__init__.py +++ b/gear/gear/__init__.py @@ -1,7 +1,7 @@ from .auth import AuthClient, maybe_parse_bearer_header from .auth_utils import create_session, insert_user from .csrf import check_csrf_token, new_csrf_token -from .database import Database, Transaction, create_database_pool, transaction +from .database import Database, Transaction, create_database_pool, resolve_test_db_endpoint, transaction from .http_server_utils import json_request, json_response from .metrics import monitor_endpoints_middleware from .session import setup_aiohttp_session @@ -21,4 +21,5 @@ 'monitor_endpoints_middleware', 'json_request', 'json_response', + 'resolve_test_db_endpoint', ] diff --git a/gear/gear/database.py b/gear/gear/database.py index fe62ce59fe9..82d0ed59dc5 100644 --- a/gear/gear/database.py +++ b/gear/gear/database.py @@ -7,11 +7,14 @@ from typing import Optional import aiomysql +import kubernetes_asyncio.client +import kubernetes_asyncio.config import pymysql from gear.metrics import DB_CONNECTION_QUEUE_SIZE, SQL_TRANSACTIONS, PrometheusSQLTimer from hailtop.aiotools import BackgroundTaskManager from hailtop.auth.sql_config import SQLConfig +from hailtop.config import get_deploy_config from hailtop.utils import sleep_and_backoff log = logging.getLogger('gear.database') @@ -77,6 +80,19 @@ async def aexit(acontext_manager, exc_type=None, exc_val=None, exc_tb=None): return await acontext_manager.__aexit__(exc_type, exc_val, exc_tb) +async def resolve_test_db_endpoint(sql_config: SQLConfig) -> SQLConfig: + service_name, namespace = sql_config.host[: -len('.svc.cluster.local')].split('.', maxsplit=1) + await kubernetes_asyncio.config.load_kube_config() + async with kubernetes_asyncio.client.ApiClient() as api: + client = kubernetes_asyncio.client.CoreV1Api(api) + db_service = await client.read_namespaced_service(service_name, namespace) + db_pod = await client.read_namespaced_pod(f'{db_service.spec.selector["app"]}-0', namespace) + sql_config_dict = sql_config.to_dict() + sql_config_dict['host'] = db_pod.status.host_ip + sql_config_dict['port'] = db_service.spec.ports[0].node_port + return SQLConfig.from_dict(sql_config_dict) + + def get_sql_config(maybe_config_file: Optional[str] = None) -> SQLConfig: if maybe_config_file is None: config_file = os.environ.get('HAIL_DATABASE_CONFIG_FILE', '/sql-config/sql-config.json') @@ -108,6 +124,8 @@ def get_database_ssl_context(sql_config: Optional[SQLConfig] = None) -> ssl.SSLC @retry_transient_mysql_errors async def create_database_pool(config_file: Optional[str] = None, autocommit: bool = True, maxsize: int = 10): sql_config = get_sql_config(config_file) + if get_deploy_config().location() != 'k8s' and sql_config.host.endswith('svc.cluster.local'): + sql_config = await resolve_test_db_endpoint(sql_config) ssl_context = get_database_ssl_context(sql_config) assert ssl_context is not None return await aiomysql.create_pool( diff --git a/hail/python/hailtop/auth/sql_config.py b/hail/python/hailtop/auth/sql_config.py index a247d3f769d..b334f54e14b 100644 --- a/hail/python/hailtop/auth/sql_config.py +++ b/hail/python/hailtop/auth/sql_config.py @@ -39,6 +39,7 @@ def to_cnf(self) -> str: cnf = f'''[client] host={self.host} user={self.user} +port={self.port} password="{self.password}" ssl-ca={self.ssl_ca} ssl-mode={self.ssl_mode} diff --git a/tls/Dockerfile b/tls/Dockerfile index 168e1b234d2..a7a9739091f 100644 --- a/tls/Dockerfile +++ b/tls/Dockerfile @@ -8,5 +8,5 @@ RUN curl -LO https://dl.k8s.io/release/v1.21.14/bin/linux/amd64/kubectl && \ sed -i 's/^RANDFILE/#RANDFILE/' /etc/ssl/openssl.cnf && \ hail-pip-install pyyaml -COPY config.yaml . -COPY create_certs.py . +COPY config.yaml / +COPY create_certs.py create_test_db_config.sh / diff --git a/tls/create_test_db_config.sh b/tls/create_test_db_config.sh new file mode 100644 index 00000000000..b5bde7a2e0c --- /dev/null +++ b/tls/create_test_db_config.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +if [ -z "${NAMESPACE}" ]; then + echo "Must specify a NAMESPACE environment variable" + exit 1; +elif [ "${NAMESPACE}" == "default" ]; then + echo "This script is only for creating test database configs" + exit 1; +fi + +function create_key_and_cert() { + local name=$1 + local key_file=${name}-key.pem + local csr_file=${name}-csr.csr + local cert_file=${name}-cert.pem + + openssl genrsa -out ${key_file} 4096 + openssl req -new -subj /CN=$name -key $key_file -out $csr_file + openssl x509 -req -in $csr_file \ + -CA server-ca.pem -CAkey server-ca-key.pem \ + -CAcreateserial \ + -out $cert_file \ + -days 365 -sha256 +} + +dir=$(mktemp -d) +cd $dir + +# Create the MySQL server CA +openssl req -new -x509 \ + -subj /CN=db-root -nodes -newkey rsa:4096 \ + -keyout server-ca-key.pem -out server-ca.pem + +create_key_and_cert server +create_key_and_cert client + +set +x +LC_ALL=C tr -dc '[:alnum:]' db-root-password +password=$(cat db-root-password) + +cat >sql-config.cnf <sql-config.json <