feat(ingest,ci): Move ncbi_api_key to secrets. (#2357)

* Move ncbi_api_key to secrets.

* Add secret as a sealed secret

* Only use sealedsecrets on preview instances (not e2e tests)

ingest/Snakefile

@@ -1,4 +1,5 @@
 import yaml
+import os
 from pathlib import Path
 
 with open("config/defaults.yaml") as f:
@@ -22,6 +23,7 @@ SEGMENTED = config["segmented"]
 ALL_FIELDS = ",".join(config["all_fields"])
 COLUMN_MAPPING = config["column_mapping"]
 LOG_LEVEL = config.get("log_level", "INFO")
+NCBI_API_KEY = os.getenv("NCBI_API_KEY")
 FILTER_FASTA_HEADERS = config.get("filter_fasta_headers", None)
 
 
@@ -56,12 +58,13 @@ rule fetch_ncbi_dataset_package:
         dataset_package="results/ncbi_dataset.zip",
     params:
         taxon_id=TAXON_ID,
+        api_key=NCBI_API_KEY
     shell:
         """
         datasets download virus genome taxon {params.taxon_id} \
             --no-progressbar \
             --filename {output.dataset_package} \
-            --api-key 15c4ff96de265753f878bb08d88ca64df909 \
+            --api-key {params.api_key} \
         """
 
 

kubernetes/loculus/templates/ingest-deployment.yaml

@@ -38,6 +38,11 @@ spec:
                 secretKeyRef:
                   name: service-accounts
                   key: insdcIngestUserPassword
+            - name: NCBI_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: ingest-ncbi
+                  key: api-key
           args:
             - snakemake
             - results/approved

kubernetes/loculus/values.yaml

@@ -1493,6 +1493,10 @@ secrets:
     type: raw
     data:
       orcidSecret: "dummy"
+  ingest-ncbi:
+    type: raw
+    data:
+      api-key: "dummy"
 enableCrossRefCredentials: true
 runDevelopmentKeycloakDatabase: true
 runDevelopmentMainDatabase: true

kubernetes/loculus/values_preview_server.yaml

@@ -15,4 +15,9 @@ secrets:
     clusterWide: "true"
     data:
       orcidSecret: AgBwVSGo2Y9TPdkGd78BikI/Nm+4SI3TS5AHQu+h4s0qb5Ydc1m7MJLd5ZwoXhkvWWfm2kt8K0Tj5eoAsFSVktA5EIZMWZOR7DugU5Z5GLgGJogpgrzN+dR7AEsyxwfXGzF2RISqsBDjRBzzfiF0ZhpyjUOCab2fn9iH3HmJ9qPK6xpzOcS47jjfg+5ofv8BxdGPAKiOpLUgFAZO1xTKZXTnbdVvyRUORJVZJffwiZ3RG3EO+eUKgHIf5YeG0b52OzDPfCzgNDGV/ryY+jPix6c+T1/g2qnUpLzECYdaqc5R6ZWtNt42VELxU3kpmwNyswraec+3IxkufXZTyTpSxikpHDKK9Zcz9woh/+R4BUKjjO1zpHWFPV5jY+hzY+rxfHTP8VX0txyMsBVEafDRL0R3YYz8rUBzXC4C6IFJLK0QmJKD7GHQM2Ue6SjGkr49k1ZPSGCGAJ16IZCe3l3lVhzncobpEBPx4qYJhD8oo+ZIt7YIZ/Egmkza+DFAAWmtSCEztL85Y3pNN5g3SVn6urSKZbNUeI6rC4JqwS6Uk2370VrvEEufml9fXOdE5ATO5C+zHj3AWd0zancDsk4nQegbE5fjLblGfQ1g/ID2OHCFaw7FMBItrE+R5VkTwN5nuszAnWXc191T7PS3mWrYBDrjbqHOr/P1X6BZWT0Q7bqR8y5d2JUX5qNaHI4s4WN43Z/m0oPRMZC0WHn/ebtPPukoKRWnMWqN8MTwnB9oHV/84egrHUY=
+  ingest-ncbi:
+    type: sealedsecret
+    clusterWide: "true"
+    data:
+      api-key: "AgAwbNNXOtyIXqZzr0SswXwp7Q48HgaafxMXvGByPpJClR+L+jFFJHgHrVnDbI+Gt/YGlETOeYhlS7qHD/erANZForlM5LegtG+0u9UunZPXtfoRmYAeCqie7adIuE6gfH7Ubf+DsGxI0DNCwaM0xNcNDohBgIwNZpigLrpHWlA/Yq95k/2mG/qFT7AUltSpPDoQPKfor0jeuVSGIiQQ9lsaUchJK1YDfLHWhKy+/ymNE+1rEerfp4wx+kMGMV4zDPbYtV9OTxne+qyddV6vBbWB2+8otHIneGs3zE6YVOAVGMvwh2w+mSJverPbzX4B705ovDsrKARbLtrVUwnvlDeqtbG3TUlI9OdcoN+PT+Lk3NyGca8jOOCQp3aU3RTi8Z9kyUa69v6QiiU3i1q8sn6F4DgRuwFvXOSPrrxvvq25RB+XXctP0y/BrPM5wiLI4l9PiHO6eEKq/ZO/df/ab493Dd8DNDm1eu+qsPb7Ju+BzorkX7CoNVTdnKZ2gPmK+k1GTLRm/til+E4MwTMo5xIjuf5lWPVgbgVeUvN5tFcz/YniS3MX3RHY1OzWG/XnsNM8b0hDxI/Rw0omqM5S9kkN+NjPYjxRbfhtuLazddDiu/RAU6zc8L4IVNXzEnmKZv3bIAH7wIfmli4XjRmtr9isrshUelOUA1OsPrhIpey+NheFShPxNVC2IZoCC9OlQ3vKzUuXL7TI+sw3/8vwVKlm3ok4OO0+AyEccnlIAx5Kygd5fkJ3txzpMdo0jvk7c8Q="
 reduceResourceRequest: true