From b100266f85a9a8fbefe3ded1af4add8b7d7e600b Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Tue, 30 Jul 2024 21:26:14 +0200 Subject: [PATCH 1/6] Move ncbi_api_key to secrets. --- ingest/Snakefile | 5 ++++- kubernetes/loculus/templates/ingest-deployment.yaml | 5 +++++ kubernetes/loculus/values.yaml | 4 ++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ingest/Snakefile b/ingest/Snakefile index 15198c7e0..4e9855c7f 100644 --- a/ingest/Snakefile +++ b/ingest/Snakefile @@ -1,4 +1,5 @@ import yaml +import os from pathlib import Path with open("config/defaults.yaml") as f: @@ -22,6 +23,7 @@ SEGMENTED = config["segmented"] ALL_FIELDS = ",".join(config["all_fields"]) COLUMN_MAPPING = config["column_mapping"] LOG_LEVEL = config.get("log_level", "INFO") +NCBI_API_KEY = os.getenv("NCBI_API_KEY") def rename_columns(input_file, output_file, mapping=COLUMN_MAPPING): @@ -55,12 +57,13 @@ rule fetch_ncbi_dataset_package: dataset_package="results/ncbi_dataset.zip", params: taxon_id=TAXON_ID, + api_key=NCBI_API_KEY shell: """ datasets download virus genome taxon {params.taxon_id} \ --no-progressbar \ --filename {output.dataset_package} \ - --api-key 15c4ff96de265753f878bb08d88ca64df909 \ + --api-key {params.api_key} \ """ diff --git a/kubernetes/loculus/templates/ingest-deployment.yaml b/kubernetes/loculus/templates/ingest-deployment.yaml index 17d951e05..00cecce64 100644 --- a/kubernetes/loculus/templates/ingest-deployment.yaml +++ b/kubernetes/loculus/templates/ingest-deployment.yaml @@ -38,6 +38,11 @@ spec: secretKeyRef: name: service-accounts key: insdcIngestUserPassword + - name: NCBI_API_KEY + valueFrom: + secretKeyRef: + name: ingest-ncbi + key: api-key args: - snakemake - results/approved diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index bf6c770dd..c8b41c741 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1460,6 +1460,10 @@ secrets: url: "jdbc:postgresql://loculus-database-service/loculus" username: "postgres" password: "password" + ingest-ncbi: + type: raw + data: + api-key: "15c4ff96de265753f878bb08d88ca64df909" keycloak-database: type: raw data: From 1036e96346c0e5140a73a90ff91907c50b9a6f28 Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:16:37 +0200 Subject: [PATCH 2/6] Add secret as a sealed secret --- kubernetes/loculus/values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 952d6bd38..2d47271d0 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1464,9 +1464,10 @@ secrets: username: "postgres" password: "password" ingest-ncbi: - type: raw + type: sealedsecret + clusterWide: "true" data: - api-key: "15c4ff96de265753f878bb08d88ca64df909" + api-key: "AgDN2pTgoAgJM5D/8XpEt2THyVyAhcGzT77wA+E3sMVH2TBZPi8TuyQVcvrl5y9bhUjRmLTtEdX2e1pjQ1zdaViXpRh/CN/L1r5IL6xvn8k/62wc9VpI3wwyRDHp9hztcyVXa1Sk/TwLF8ijiiajQI+b4uJd/qQkIrLgW++aJS4Jow9cDR3ETDfpxbbyIDVXv8DIIWn72V/Yvk582j+dmQ04WQIWUhLLyuubS6wnzAZ70yvRdpEwzq0kPScPIBu3VANwiaPg5Mh5FjMbtMqDF4aQQgWblT/aQ0hMy6LwLxP9dh+R7ZnpEUNj5PRqmdAlunYKGLfO7/23x46tlZYqR+oAfFYKmdFkXmpL+/GaaBjR0Mk7cz+CBI9uM5w6XEkE1sb4Os0KaCT7gtRkrtU7Q9k53TqJ5ojC6KAiVtWUCFpQxvQbtJiapNeGsuuGqg97jDk5oNOW9JVLcGayDJMhYt8brPKs0/ozzPV+rHRMZv28SeHh6wNd5QqJv331+smjrxcXnUP92Klcj65cgwf0++/FxXD68ectNMSxrNdFzwVuQifomWihvGfGXBDWoWRchyny3PP+/mVMVZa4TtXMdJ+ne6Ya/WPxk2N35FjX6VwCjrzxRksWatvtbHRQbBozWrmlcKwGv35MNUpKKoTKJYTlAZksgCukND6b2GQRZ9ss2Fw7qQoUOMUCwX0LLMTW7F774IK8AxffN9Js6mw1DV9WrVYgdYD9foJ/oLxb92gnkRMFXEc=" keycloak-database: type: raw data: From 55f9303287d7d9ee9b497530cc81cd2cef80f004 Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:21:14 +0200 Subject: [PATCH 3/6] check if changing to namespace: default works --- kubernetes/loculus/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 2d47271d0..c9d485d7c 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1467,7 +1467,7 @@ secrets: type: sealedsecret clusterWide: "true" data: - api-key: "AgDN2pTgoAgJM5D/8XpEt2THyVyAhcGzT77wA+E3sMVH2TBZPi8TuyQVcvrl5y9bhUjRmLTtEdX2e1pjQ1zdaViXpRh/CN/L1r5IL6xvn8k/62wc9VpI3wwyRDHp9hztcyVXa1Sk/TwLF8ijiiajQI+b4uJd/qQkIrLgW++aJS4Jow9cDR3ETDfpxbbyIDVXv8DIIWn72V/Yvk582j+dmQ04WQIWUhLLyuubS6wnzAZ70yvRdpEwzq0kPScPIBu3VANwiaPg5Mh5FjMbtMqDF4aQQgWblT/aQ0hMy6LwLxP9dh+R7ZnpEUNj5PRqmdAlunYKGLfO7/23x46tlZYqR+oAfFYKmdFkXmpL+/GaaBjR0Mk7cz+CBI9uM5w6XEkE1sb4Os0KaCT7gtRkrtU7Q9k53TqJ5ojC6KAiVtWUCFpQxvQbtJiapNeGsuuGqg97jDk5oNOW9JVLcGayDJMhYt8brPKs0/ozzPV+rHRMZv28SeHh6wNd5QqJv331+smjrxcXnUP92Klcj65cgwf0++/FxXD68ectNMSxrNdFzwVuQifomWihvGfGXBDWoWRchyny3PP+/mVMVZa4TtXMdJ+ne6Ya/WPxk2N35FjX6VwCjrzxRksWatvtbHRQbBozWrmlcKwGv35MNUpKKoTKJYTlAZksgCukND6b2GQRZ9ss2Fw7qQoUOMUCwX0LLMTW7F774IK8AxffN9Js6mw1DV9WrVYgdYD9foJ/oLxb92gnkRMFXEc=" + api-key: "AgAY8630XySvcoonNQJdtA5L1vQ5wYC9kfJJXeOiFp1sxEqhNdcTLNVL3bSUr75zBBS3wP99qVFB5fvMHtCCroB3RZZBpwwBxQfkb0UBYG9LfGXmp1/m8jvQm+AXW12VaoNfd8QNTgNB63fTGeH25jHr6w09qqpBBEC6CntzpkO0bY90XzHtdcn3AUwxnZ0vXxNQ3KEZ9uX6AUqXhJkn0XhUx3h2HOQf2On4tNB2ozSEWekjBoyhUz8HcIEFMsap1piTptc7zH9vgAfmYCuE+CIUd45/BH69bHlQwkMGeZaMR1LHheLw/gHQHu06XLWCt9URbvUdN1uC2rBBBcE9/YJZ+R7Newrfij9KPMHEBHhvqIPHOC+Fr44sFcGEu1EIjh31pDI0kZub9y2vetA1+dUBfNhhsG0XJ+8TX0fSQJz2qLWyJFV+9QQj8JhPZTXQhd517volBFz6UdLNajbyjgB24kCKe6IDmS+4URIFYZaYYKrCL4qe6dyMM+HxmNWs7/FuzdGgPD/M4NUpBENJOEjjWDpL0yGDO+fdgXQ/g/W3BzUG+nfTYA8tv/T0Tt+fUJXc32nz494jbFLeZ7KDYeHlhfJO9FiPHLnBgkgbe3ufv2AlvfW74uyLApoB7rikPV1bN5g5IjQZ6lr/3WTcgg6RSZYNnLFZrGPA14dX0kmJ3JzsdHfpowj2yNPIN0KSU+zXjQRV5U/fJxYlfCS6VSW5peeOshg7YNE+IQU5dshign7mv8M=" keycloak-database: type: raw data: From d786b3ea3751b76f1a419b44a44fa1f9ebf79dd1 Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:30:47 +0200 Subject: [PATCH 4/6] Try again by creating secret using kubectl create secret generic ingest-ncbi --from-literal=api-key=KEY --dry-run=client -o yaml > secret.yaml --- kubernetes/loculus/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index c9d485d7c..ffed103fd 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1467,7 +1467,7 @@ secrets: type: sealedsecret clusterWide: "true" data: - api-key: "AgAY8630XySvcoonNQJdtA5L1vQ5wYC9kfJJXeOiFp1sxEqhNdcTLNVL3bSUr75zBBS3wP99qVFB5fvMHtCCroB3RZZBpwwBxQfkb0UBYG9LfGXmp1/m8jvQm+AXW12VaoNfd8QNTgNB63fTGeH25jHr6w09qqpBBEC6CntzpkO0bY90XzHtdcn3AUwxnZ0vXxNQ3KEZ9uX6AUqXhJkn0XhUx3h2HOQf2On4tNB2ozSEWekjBoyhUz8HcIEFMsap1piTptc7zH9vgAfmYCuE+CIUd45/BH69bHlQwkMGeZaMR1LHheLw/gHQHu06XLWCt9URbvUdN1uC2rBBBcE9/YJZ+R7Newrfij9KPMHEBHhvqIPHOC+Fr44sFcGEu1EIjh31pDI0kZub9y2vetA1+dUBfNhhsG0XJ+8TX0fSQJz2qLWyJFV+9QQj8JhPZTXQhd517volBFz6UdLNajbyjgB24kCKe6IDmS+4URIFYZaYYKrCL4qe6dyMM+HxmNWs7/FuzdGgPD/M4NUpBENJOEjjWDpL0yGDO+fdgXQ/g/W3BzUG+nfTYA8tv/T0Tt+fUJXc32nz494jbFLeZ7KDYeHlhfJO9FiPHLnBgkgbe3ufv2AlvfW74uyLApoB7rikPV1bN5g5IjQZ6lr/3WTcgg6RSZYNnLFZrGPA14dX0kmJ3JzsdHfpowj2yNPIN0KSU+zXjQRV5U/fJxYlfCS6VSW5peeOshg7YNE+IQU5dshign7mv8M=" + api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=" keycloak-database: type: raw data: From 85270a7d7a05ecba262a936642bed566c26374af Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:38:09 +0200 Subject: [PATCH 5/6] Only use sealedsecrets on preview instances (not e2e tests) --- kubernetes/loculus/values.yaml | 9 ++++----- kubernetes/loculus/values_preview_server.yaml | 7 ++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index ffed103fd..955f61ce5 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -1463,11 +1463,6 @@ secrets: url: "jdbc:postgresql://loculus-database-service/loculus" username: "postgres" password: "password" - ingest-ncbi: - type: sealedsecret - clusterWide: "true" - data: - api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=" keycloak-database: type: raw data: @@ -1498,6 +1493,10 @@ secrets: type: raw data: orcidSecret: "dummy" + ingest-ncbi: + type: raw + data: + api-key: "dummy" enableCrossRefCredentials: true runDevelopmentKeycloakDatabase: true runDevelopmentMainDatabase: true diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index f9081054d..ea9e79945 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -15,4 +15,9 @@ secrets: clusterWide: "true" data: orcidSecret: AgBwVSGo2Y9TPdkGd78BikI/Nm+4SI3TS5AHQu+h4s0qb5Ydc1m7MJLd5ZwoXhkvWWfm2kt8K0Tj5eoAsFSVktA5EIZMWZOR7DugU5Z5GLgGJogpgrzN+dR7AEsyxwfXGzF2RISqsBDjRBzzfiF0ZhpyjUOCab2fn9iH3HmJ9qPK6xpzOcS47jjfg+5ofv8BxdGPAKiOpLUgFAZO1xTKZXTnbdVvyRUORJVZJffwiZ3RG3EO+eUKgHIf5YeG0b52OzDPfCzgNDGV/ryY+jPix6c+T1/g2qnUpLzECYdaqc5R6ZWtNt42VELxU3kpmwNyswraec+3IxkufXZTyTpSxikpHDKK9Zcz9woh/+R4BUKjjO1zpHWFPV5jY+hzY+rxfHTP8VX0txyMsBVEafDRL0R3YYz8rUBzXC4C6IFJLK0QmJKD7GHQM2Ue6SjGkr49k1ZPSGCGAJ16IZCe3l3lVhzncobpEBPx4qYJhD8oo+ZIt7YIZ/Egmkza+DFAAWmtSCEztL85Y3pNN5g3SVn6urSKZbNUeI6rC4JqwS6Uk2370VrvEEufml9fXOdE5ATO5C+zHj3AWd0zancDsk4nQegbE5fjLblGfQ1g/ID2OHCFaw7FMBItrE+R5VkTwN5nuszAnWXc191T7PS3mWrYBDrjbqHOr/P1X6BZWT0Q7bqR8y5d2JUX5qNaHI4s4WN43Z/m0oPRMZC0WHn/ebtPPukoKRWnMWqN8MTwnB9oHV/84egrHUY= -reduceResourceRequest: true + ingest-ncbi: + type: sealedsecret + clusterWide: "true" + data: + api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=" + reduceResourceRequest: true From 6023696c596bd310fd2d9180817cccebd173c554 Mon Sep 17 00:00:00 2001 From: "Anna (Anya) Parker" <50943381+anna-parker@users.noreply.github.com> Date: Mon, 5 Aug 2024 11:45:08 +0200 Subject: [PATCH 6/6] Remove incorrectly added indent --- kubernetes/loculus/values_preview_server.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/values_preview_server.yaml b/kubernetes/loculus/values_preview_server.yaml index ea9e79945..7f48e0360 100644 --- a/kubernetes/loculus/values_preview_server.yaml +++ b/kubernetes/loculus/values_preview_server.yaml @@ -20,4 +20,4 @@ secrets: clusterWide: "true" data: api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q=" - reduceResourceRequest: true +reduceResourceRequest: true