feat(ena-submission): Add ena slack notifications (#2396)

* Use slack api to send notifications when the ENA submission pipeline has found sequences to submit to ENA.

* Add slack api tokens as sealed secrets.

* Fix db connection.

ena-submission/config/config.yaml

@@ -19,7 +19,13 @@ organisms:
         name: ncbi_release_date
         type: date
       - externalMetadataUpdater: ena
-        name: ncbi_update_date
+        name: ncbi_update_date_L
+        type: date
+      - externalMetadataUpdater: ena
+        name: ncbi_update_date_M
+        type: date
+      - externalMetadataUpdater: ena
+        name: ncbi_update_date_S
         type: date
       - externalMetadataUpdater: ena
         name: ncbi_submitter_country
@@ -52,22 +58,10 @@ organisms:
         name: insdc_accession_full_S
         type: string
       - externalMetadataUpdater: ena
-        name: bioproject_accessions_L
-        type: string
-      - externalMetadataUpdater: ena
-        name: bioproject_accessions_M
-        type: string
-      - externalMetadataUpdater: ena
-        name: bioproject_accessions_S
-        type: string
-      - externalMetadataUpdater: ena
-        name: biosample_accession_L
-        type: string
-      - externalMetadataUpdater: ena
-        name: biosample_accession_M
+        name: bioproject_accessions
         type: string
       - externalMetadataUpdater: ena
-        name: biosample_accession_S
+        name: biosample_accession
         type: string
       - externalMetadataUpdater: ena
         name: ncbi_sourcedb
@@ -79,7 +73,13 @@ organisms:
         name: ncbi_virus_tax_id
         type: int
       - externalMetadataUpdater: ena
-        name: sra_run_accession
+        name: sra_run_accession_L
+        type: string
+      - externalMetadataUpdater: ena
+        name: sra_run_accession_M
+        type: string
+      - externalMetadataUpdater: ena
+        name: sra_run_accession_S
         type: string
   ebola-sudan:
     ingest:

ena-submission/environment.yml

@@ -14,3 +14,4 @@ dependencies:
   - snakemake
   - unzip
   - psycopg2
+  - slack_sdk

ena-submission/scripts/get_ena_submission_list.py

@@ -1,12 +1,14 @@
 import json
 import logging
+import os
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any, Dict, List
 
 import click
 import yaml
 from call_loculus import get_released_data
+from notifications import get_slack_config, notify, upload_file_with_comment
 from submission_db import get_db_config, in_submission_table
 
 logger = logging.getLogger(__name__)
@@ -32,6 +34,9 @@ class Config:
     db_username: str
     db_password: str
     db_host: str
+    slack_hook: str
+    slack_token: str
+    slack_channel_id: str
 
 
 def get_data_for_submission(config, entries, db_config):
@@ -62,10 +67,30 @@ def get_data_for_submission(config, entries, db_config):
             )
             continue
         data_dict[key] = item
-
     return data_dict
 
 
+def send_slack_notification(config: Config, output_file: str):
+    slack_config = get_slack_config(
+        slack_hook_default=config.slack_hook,
+        slack_token_default=config.slack_token,
+        slack_channel_id_default=config.slack_channel_id,
+    )
+    if not slack_config.slack_hook:
+        logging.info("Could not find slack hook cannot send message")
+
+    if slack_config.slack_hook:
+        comment = (
+            f"{config.backend_url}: ENA Submission pipeline wants to submit the following sequences"
+        )
+        try:
+            response = upload_file_with_comment(slack_config, output_file, comment)
+            if not response.get("ok", False):
+                raise Exception
+        except Exception as e:
+            notify(slack_config, comment + f" - file upload to slack failed with Error {e}")
+
+
 @click.command()
 @click.option(
     "--log-level",
@@ -110,10 +135,11 @@ def get_ena_submission_list(log_level, config_file, output_file):
         logging.info(f"Getting released sequences for organism: {organism}")
 
         all_entries = get_released_data(config, organism)
-        entries_to_submit = get_data_for_submission(config, all_entries, db_config)
+        entries_to_submit.update(get_data_for_submission(config, all_entries, db_config))
 
     if entries_to_submit:
         Path(output_file).write_text(json.dumps(entries_to_submit))
+        send_slack_notification(config, output_file)
     else:
         logging.info("No sequences found to submit to ENA")
         Path(output_file).write_text("")

ena-submission/scripts/notifications.py

@@ -0,0 +1,63 @@
+import json
+import logging
+import os
+import zipfile
+from dataclasses import dataclass
+
+import requests
+from slack_sdk import WebClient
+
+
+@dataclass
+class SlackConfig:
+    slack_hook: str
+    slack_token: str
+    slack_channel_id: str
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_slack_config(
+    slack_hook_default: str, slack_token_default: str, slack_channel_id_default: str
+):
+    slack_hook = os.getenv("SLACK_HOOK")
+    if not slack_hook:
+        slack_hook = slack_hook_default
+
+    slack_token = os.getenv("SLACK_TOKEN")
+    if not slack_token:
+        slack_token = slack_token_default
+
+    slack_channel_id = os.getenv("SLACK_CHANNEL_ID")
+    if not slack_channel_id:
+        slack_channel_id = slack_channel_id_default
+
+    params = {
+        "slack_hook": slack_hook,
+        "slack_token": slack_token,
+        "slack_channel_id": slack_channel_id,
+    }
+
+    return SlackConfig(**params)
+
+
+def notify(config: SlackConfig, text: str):
+    """Send slack notification using slack hook"""
+    if config.slack_hook:
+        requests.post(config.slack_hook, data=json.dumps({"text": text}), timeout=10)
+
+
+def upload_file_with_comment(config: SlackConfig, file_path: str, comment: str):
+    """Upload file with comment to slack channel"""
+    client = WebClient(token=config.slack_token)
+    output_file_zip = file_path.split(".")[0] + ".zip"
+    zip = zipfile.ZipFile(output_file_zip, "w", zipfile.ZIP_DEFLATED)
+    zip.write(file_path)
+    zip.close()
+    return client.files_upload_v2(
+        file=output_file_zip,
+        title=file_path.split("/")[-1],
+        channel=config.slack_channel_id,
+        initial_comment=comment,
+    )

kubernetes/loculus/templates/ena-submission-deployment.yaml

@@ -67,8 +67,8 @@ spec:
               valueFrom:
                 secretKeyRef:
                   name: database
-                  key: url
-            - name: DB_USER
+                  key: host
+            - name: DB_USERNAME
               valueFrom:
                 secretKeyRef:
                   name: database
@@ -78,6 +78,21 @@ spec:
                 secretKeyRef:
                   name: database
                   key: password
+            - name: SLACK_HOOK
+              valueFrom:
+                secretKeyRef:
+                  name: slack-notifications
+                  key: slack-hook
+            - name: SLACK_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: slack-notifications
+                  key: slack-token
+            - name: SLACK_CHANNEL_ID
+              valueFrom:
+                secretKeyRef:
+                  name: slack-notifications
+                  key: slack-channel-id
           args:
             - snakemake
             - get_ena_submission_list_and_sleep # Do not start submission process yet until on pods until better tested. 

kubernetes/loculus/values.yaml

@@ -1464,6 +1464,7 @@ secrets:
       url: "jdbc:postgresql://loculus-database-service/loculus"
       username: "postgres"
       password: "password"
+      host: "loculus-database-service"
   keycloak-database:
     type: raw
     data:
@@ -1498,6 +1499,12 @@ secrets:
     type: raw
     data:
       api-key: "dummy"
+  slack-notifications:
+    type: raw
+    data:
+      slack-hook: "dummy"
+      slack-token: "dummy"
+      slack-channel-id: "dummy"
 enableCrossRefCredentials: true
 runDevelopmentKeycloakDatabase: true
 runDevelopmentMainDatabase: true

kubernetes/loculus/values_preview_server.yaml

@@ -19,5 +19,12 @@ secrets:
     type: sealedsecret
     clusterWide: "true"
     data:
-      api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q="
+      api-key: AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=
+  slack-notifications:
+    type: sealedsecret
+    clusterWide: "true"
+    data:
+      slack-hook: AgCLEhTwqKL278AbNwpqdRqeg6naNrQJWx3q8Zp+ecXjMaaLLBi1C3uQlt0WKioy+pUAhfe7MowXKLM55hLyh/InZ9o3yLi9T/5cVRXcEXCvODWmbhr94XhcYI3KnVngZLcNl9Gr4LR+bz8A0sl/rCijNYrqYeDCLI6XUmB8mlKnHPqrF6CXC8Y5xyDbNYJONx6DAugq+gQcZYJ101vUOtu2LTD8awCsdF5FOzdcZ344Vxn/xwDlbbvUEKEQp5A5aMfx95zpa+rV/sQYHeCb7Dy1oWqpOrZP/rPJ4K9VGRx5QA9o1Qi0Pl3alRUqiPUR6pbMxbX8u0kCN6drFKxXDAMd+SadsppDGbNQNeQP5cphNJwYxL/0MIgXxJTrQpcynJK1FULX9W+1GtXg+tX4hRCtZL5hnCxPw12QcNOL2N8SJLGEe8gK8QtALpu/DH/trVJ3rMDRkZhhWCvtb9Zt9EuvUhxs07sE9DZ7rEAqzx51v4vr9CzmxkHEiAhrC3Se3CxnSspBP1/X9SvZ+GXn+ZuXzN+KivWCnim0RwhRD75Y7ZP8ct/iu3ilb6b7Pl+KOgOkA3In7c4yVAZwXMTuF6aP2/8inPx5Kk6p8ks7c5XeSIDFOH7C6EJuD7E69Fz6ijaF5bJN8NWBVxkE88xq4un5e7dcuqjIqaQ8kLDX1g8aXiklr6qD29q9H/m+gtd6lxcMb53bMl0EI+GHYTBZn3w+T1PxlY1uoBfNzt1efjXJD7AWlTDxze+5PIYgiFVAOdfv3ey5HJMMw1w5MTLMW44hkpt4MCaHvREBTXq5sxBJe10=
+      slack-token: AgDFAP+F7ze+TY+yK71JPOSkKwIpnBXh7WWweyZZwYm1/CwmKS1iQ4O1p54sTrHrMC737Si27MfTVvrEZRj9aAwq0fQJZt5yldpfMaTccZJHj3rQ5kczyuCMYVcFzmyywAr4DXsUCscjrOLgwTiiK/d8jSFkKyXupC7bB2EcZZUFGpFAj7PiSeJuEIKQptkSjeaasCQXkdkuoKczM38GCu502pIxaJ9kIXVrereyKUpsU/uFDgj0IcKqfiT0M2FGs8Ujl3CXpMxcOLSuxVyCnje27GHpsYrd/uEKX3yl1rB1rV6Z+gMzlO9DDPW/XJl6TY5snOxdaCv7uNzAGwgb/rlaZ5fnrNqsOoucJvh35yxMcKDsx/hY72H7PRnzNpLeqZ/2zAub+fQP/o1edjxaYHaSltS0lwzCivIPOHv66/dDOD9v0LncWkCWGXXOp8Fsz9OrF+NcAZjIY/hDzwy+JRDA7Wtn7jlkA07WFpobkyyKfN+bNT1664wS2IMDRYA6+MbkA99v+ScVsEVlxJqn+PiaDtexQAfQcyN2NPbQe+9xMIQavvrcnwxdMwAP8DBME3vhdrD8yDRJ9GN+ygtZ3dB4FC4iW20ETyzlAqJ/H9M2/ed1O3VyraFDCV3PmSBdq7Rinj1Zg6D+IEp54HtwWiu5s7iNeKW37cSSloRUaojWQ1BFPB6msfP/O5yqREdGrWVhmChWvSDMw2LxmnZbCw3mVdMr9B1XeK76GHa4kOhVOcEqzl0X61NILYDecgLP6HVJZhB+NpHJfOY=
+      slack-channel-id: AgDDxu0CGC/AFqOHMeHjV3KUGPoY0QAmGoqtiuxPPaP+GWOuz4xZbBP6Fymh0XbHFHMB9PtWowrMbvFOkfTncRiKqyK6HIU7GU2GtCla2WTZQNRAW82gcJnxtbtm4KymN3LyTj27qBHkQHNZ90qyBGdsJUYwmBW8XB6wNoz10KDyQSvYrYwQEe0onCgnislxslATPB6CQFWHMghKYoaHHAECpf6sN1kS9rvNq/1e7gG8s++lgF7qSZgjQP77Q6kMoiMS5krX03pPKZXsc69mI8GLIvhalsUfg2BO7swa3FgCbjecp32lW6KuRCfzeMmj2NWpWTLcJSPYPJN0sOkhRFOWUlrylztG82l7dGl5JofJWQVopF+qLTAR6LxHaujFHQ+Y2x4/5tBmurwOT6xXknQjXqYs2qbG1OriivJrjRwhRoZWE2vR5YlE+Zz8S9/vYw0JnKibnB1YvbdBBnpllyXYjTJa4818W05DvJ70qLeILMYcEkY4/jv5xqNdGuwp34gZLcW6+qztHfQVXRf0uXM1B7BPNH0aNMBNN7D0m1vWkTNgKC+V2PiEH9nTVhwSF+MlG/rmR2+v84kWhMP3qdX8/28GnBnvS9rryzuc3e2mHBIiIGHwW+SQjlmdq0jDTtuvFtU8I7ncB5PUe+sYZ0zFrn57blraBG5ntqtZfb+aS3modE+ElmCgBzi8gSQoVxXzmOIfMZRRwAaH0w==
 reduceResourceRequest: true