diff --git a/ena-submission/config/config.yaml b/ena-submission/config/config.yaml index 59512add0..7eb1e2326 100644 --- a/ena-submission/config/config.yaml +++ b/ena-submission/config/config.yaml @@ -19,7 +19,13 @@ organisms: name: ncbi_release_date type: date - externalMetadataUpdater: ena - name: ncbi_update_date + name: ncbi_update_date_L + type: date + - externalMetadataUpdater: ena + name: ncbi_update_date_M + type: date + - externalMetadataUpdater: ena + name: ncbi_update_date_S type: date - externalMetadataUpdater: ena name: ncbi_submitter_country @@ -52,22 +58,10 @@ organisms: name: insdc_accession_full_S type: string - externalMetadataUpdater: ena - name: bioproject_accessions_L - type: string - - externalMetadataUpdater: ena - name: bioproject_accessions_M - type: string - - externalMetadataUpdater: ena - name: bioproject_accessions_S - type: string - - externalMetadataUpdater: ena - name: biosample_accession_L - type: string - - externalMetadataUpdater: ena - name: biosample_accession_M + name: bioproject_accessions type: string - externalMetadataUpdater: ena - name: biosample_accession_S + name: biosample_accession type: string - externalMetadataUpdater: ena name: ncbi_sourcedb @@ -79,7 +73,13 @@ organisms: name: ncbi_virus_tax_id type: int - externalMetadataUpdater: ena - name: sra_run_accession + name: sra_run_accession_L + type: string + - externalMetadataUpdater: ena + name: sra_run_accession_M + type: string + - externalMetadataUpdater: ena + name: sra_run_accession_S type: string ebola-sudan: ingest: diff --git a/ena-submission/environment.yml b/ena-submission/environment.yml index eefb88285..5105f5cff 100644 --- a/ena-submission/environment.yml +++ b/ena-submission/environment.yml @@ -14,3 +14,4 @@ dependencies: - snakemake - unzip - psycopg2 + - slack_sdk diff --git a/ena-submission/scripts/get_ena_submission_list.py b/ena-submission/scripts/get_ena_submission_list.py index 3d739a432..3421bdd8f 100644 --- a/ena-submission/scripts/get_ena_submission_list.py +++ b/ena-submission/scripts/get_ena_submission_list.py @@ -1,5 +1,6 @@ import json import logging +import os from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List @@ -7,6 +8,7 @@ import click import yaml from call_loculus import get_released_data +from notifications import get_slack_config, notify, upload_file_with_comment from submission_db import get_db_config, in_submission_table logger = logging.getLogger(__name__) @@ -32,6 +34,9 @@ class Config: db_username: str db_password: str db_host: str + slack_hook: str + slack_token: str + slack_channel_id: str def get_data_for_submission(config, entries, db_config): @@ -62,10 +67,30 @@ def get_data_for_submission(config, entries, db_config): ) continue data_dict[key] = item - return data_dict +def send_slack_notification(config: Config, output_file: str): + slack_config = get_slack_config( + slack_hook_default=config.slack_hook, + slack_token_default=config.slack_token, + slack_channel_id_default=config.slack_channel_id, + ) + if not slack_config.slack_hook: + logging.info("Could not find slack hook cannot send message") + + if slack_config.slack_hook: + comment = ( + f"{config.backend_url}: ENA Submission pipeline wants to submit the following sequences" + ) + try: + response = upload_file_with_comment(slack_config, output_file, comment) + if not response.get("ok", False): + raise Exception + except Exception as e: + notify(slack_config, comment + f" - file upload to slack failed with Error {e}") + + @click.command() @click.option( "--log-level", @@ -110,10 +135,11 @@ def get_ena_submission_list(log_level, config_file, output_file): logging.info(f"Getting released sequences for organism: {organism}") all_entries = get_released_data(config, organism) - entries_to_submit = get_data_for_submission(config, all_entries, db_config) + entries_to_submit.update(get_data_for_submission(config, all_entries, db_config)) if entries_to_submit: Path(output_file).write_text(json.dumps(entries_to_submit)) + send_slack_notification(config, output_file) else: logging.info("No sequences found to submit to ENA") Path(output_file).write_text("") diff --git a/ena-submission/scripts/notifications.py b/ena-submission/scripts/notifications.py new file mode 100644 index 000000000..302b2f305 --- /dev/null +++ b/ena-submission/scripts/notifications.py @@ -0,0 +1,63 @@ +import json +import logging +import os +import zipfile +from dataclasses import dataclass + +import requests +from slack_sdk import WebClient + + +@dataclass +class SlackConfig: + slack_hook: str + slack_token: str + slack_channel_id: str + + +logger = logging.getLogger(__name__) + + +def get_slack_config( + slack_hook_default: str, slack_token_default: str, slack_channel_id_default: str +): + slack_hook = os.getenv("SLACK_HOOK") + if not slack_hook: + slack_hook = slack_hook_default + + slack_token = os.getenv("SLACK_TOKEN") + if not slack_token: + slack_token = slack_token_default + + slack_channel_id = os.getenv("SLACK_CHANNEL_ID") + if not slack_channel_id: + slack_channel_id = slack_channel_id_default + + params = { + "slack_hook": slack_hook, + "slack_token": slack_token, + "slack_channel_id": slack_channel_id, + } + + return SlackConfig(**params) + + +def notify(config: SlackConfig, text: str): + """Send slack notification using slack hook""" + if config.slack_hook: + requests.post(config.slack_hook, data=json.dumps({"text": text}), timeout=10) + + +def upload_file_with_comment(config: SlackConfig, file_path: str, comment: str): + """Upload file with comment to slack channel""" + client = WebClient(token=config.slack_token) + output_file_zip = file_path.split(".")[0] + ".zip" + zip = zipfile.ZipFile(output_file_zip, "w", zipfile.ZIP_DEFLATED) + zip.write(file_path) + zip.close() + return client.files_upload_v2( + file=output_file_zip, + title=file_path.split("/")[-1], + channel=config.slack_channel_id, + initial_comment=comment, + ) diff --git a/kubernetes/loculus/templates/ena-submission-deployment.yaml b/kubernetes/loculus/templates/ena-submission-deployment.yaml index 6c0ee5fee..999766b0d 100644 --- a/kubernetes/loculus/templates/ena-submission-deployment.yaml +++ b/kubernetes/loculus/templates/ena-submission-deployment.yaml @@ -67,8 +67,8 @@ spec: valueFrom: secretKeyRef: name: database - key: url - - name: DB_USER + key: host + - name: DB_USERNAME valueFrom: secretKeyRef: name: database @@ -78,6 +78,21 @@ spec: secretKeyRef: name: database key: password + - name: SLACK_HOOK + valueFrom: + secretKeyRef: + name: slack-notifications + key: slack-hook + - name: SLACK_TOKEN + valueFrom: + secretKeyRef: + name: slack-notifications + key: slack-token + - name: SLACK_CHANNEL_ID + valueFrom: + secretKeyRef: + name: slack-notifications + key: slack-channel-id args: - snakemake - get_ena_submission_list_and_sleep # Do not start submission process yet until on pods until better tested. diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index a9b236e5c..82c9233e8 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1464,6 +1464,7 @@ secrets: url: "jdbc:postgresql://loculus-database-service/loculus" username: "postgres" password: "password" + host: "loculus-database-service" keycloak-database: type: raw data: @@ -1498,6 +1499,12 @@ secrets: type: raw data: api-key: "dummy" + slack-notifications: + type: raw + data: + slack-hook: "dummy" + slack-token: "dummy" + slack-channel-id: "dummy" enableCrossRefCredentials: true runDevelopmentKeycloakDatabase: true runDevelopmentMainDatabase: true diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index 7f48e0360..6bac7aac8 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -19,5 +19,12 @@ secrets: type: sealedsecret clusterWide: "true" data: - api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=" + api-key: AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q= + slack-notifications: + type: sealedsecret + clusterWide: "true" + data: + slack-hook: AgCLEhTwqKL278AbNwpqdRqeg6naNrQJWx3q8Zp+ecXjMaaLLBi1C3uQlt0WKioy+pUAhfe7MowXKLM55hLyh/InZ9o3yLi9T/5cVRXcEXCvODWmbhr94XhcYI3KnVngZLcNl9Gr4LR+bz8A0sl/rCijNYrqYeDCLI6XUmB8mlKnHPqrF6CXC8Y5xyDbNYJONx6DAugq+gQcZYJ101vUOtu2LTD8awCsdF5FOzdcZ344Vxn/xwDlbbvUEKEQp5A5aMfx95zpa+rV/sQYHeCb7Dy1oWqpOrZP/rPJ4K9VGRx5QA9o1Qi0Pl3alRUqiPUR6pbMxbX8u0kCN6drFKxXDAMd+SadsppDGbNQNeQP5cphNJwYxL/0MIgXxJTrQpcynJK1FULX9W+1GtXg+tX4hRCtZL5hnCxPw12QcNOL2N8SJLGEe8gK8QtALpu/DH/trVJ3rMDRkZhhWCvtb9Zt9EuvUhxs07sE9DZ7rEAqzx51v4vr9CzmxkHEiAhrC3Se3CxnSspBP1/X9SvZ+GXn+ZuXzN+KivWCnim0RwhRD75Y7ZP8ct/iu3ilb6b7Pl+KOgOkA3In7c4yVAZwXMTuF6aP2/8inPx5Kk6p8ks7c5XeSIDFOH7C6EJuD7E69Fz6ijaF5bJN8NWBVxkE88xq4un5e7dcuqjIqaQ8kLDX1g8aXiklr6qD29q9H/m+gtd6lxcMb53bMl0EI+GHYTBZn3w+T1PxlY1uoBfNzt1efjXJD7AWlTDxze+5PIYgiFVAOdfv3ey5HJMMw1w5MTLMW44hkpt4MCaHvREBTXq5sxBJe10= + slack-token: AgDFAP+F7ze+TY+yK71JPOSkKwIpnBXh7WWweyZZwYm1/CwmKS1iQ4O1p54sTrHrMC737Si27MfTVvrEZRj9aAwq0fQJZt5yldpfMaTccZJHj3rQ5kczyuCMYVcFzmyywAr4DXsUCscjrOLgwTiiK/d8jSFkKyXupC7bB2EcZZUFGpFAj7PiSeJuEIKQptkSjeaasCQXkdkuoKczM38GCu502pIxaJ9kIXVrereyKUpsU/uFDgj0IcKqfiT0M2FGs8Ujl3CXpMxcOLSuxVyCnje27GHpsYrd/uEKX3yl1rB1rV6Z+gMzlO9DDPW/XJl6TY5snOxdaCv7uNzAGwgb/rlaZ5fnrNqsOoucJvh35yxMcKDsx/hY72H7PRnzNpLeqZ/2zAub+fQP/o1edjxaYHaSltS0lwzCivIPOHv66/dDOD9v0LncWkCWGXXOp8Fsz9OrF+NcAZjIY/hDzwy+JRDA7Wtn7jlkA07WFpobkyyKfN+bNT1664wS2IMDRYA6+MbkA99v+ScVsEVlxJqn+PiaDtexQAfQcyN2NPbQe+9xMIQavvrcnwxdMwAP8DBME3vhdrD8yDRJ9GN+ygtZ3dB4FC4iW20ETyzlAqJ/H9M2/ed1O3VyraFDCV3PmSBdq7Rinj1Zg6D+IEp54HtwWiu5s7iNeKW37cSSloRUaojWQ1BFPB6msfP/O5yqREdGrWVhmChWvSDMw2LxmnZbCw3mVdMr9B1XeK76GHa4kOhVOcEqzl0X61NILYDecgLP6HVJZhB+NpHJfOY= + slack-channel-id: AgDDxu0CGC/AFqOHMeHjV3KUGPoY0QAmGoqtiuxPPaP+GWOuz4xZbBP6Fymh0XbHFHMB9PtWowrMbvFOkfTncRiKqyK6HIU7GU2GtCla2WTZQNRAW82gcJnxtbtm4KymN3LyTj27qBHkQHNZ90qyBGdsJUYwmBW8XB6wNoz10KDyQSvYrYwQEe0onCgnislxslATPB6CQFWHMghKYoaHHAECpf6sN1kS9rvNq/1e7gG8s++lgF7qSZgjQP77Q6kMoiMS5krX03pPKZXsc69mI8GLIvhalsUfg2BO7swa3FgCbjecp32lW6KuRCfzeMmj2NWpWTLcJSPYPJN0sOkhRFOWUlrylztG82l7dGl5JofJWQVopF+qLTAR6LxHaujFHQ+Y2x4/5tBmurwOT6xXknQjXqYs2qbG1OriivJrjRwhRoZWE2vR5YlE+Zz8S9/vYw0JnKibnB1YvbdBBnpllyXYjTJa4818W05DvJ70qLeILMYcEkY4/jv5xqNdGuwp34gZLcW6+qztHfQVXRf0uXM1B7BPNH0aNMBNN7D0m1vWkTNgKC+V2PiEH9nTVhwSF+MlG/rmR2+v84kWhMP3qdX8/28GnBnvS9rryzuc3e2mHBIiIGHwW+SQjlmdq0jDTtuvFtU8I7ncB5PUe+sYZ0zFrn57blraBG5ntqtZfb+aS3modE+ElmCgBzi8gSQoVxXzmOIfMZRRwAaH0w== reduceResourceRequest: true