From 2c4d5bd8bc155c759ac62eaa9e6b0fdfffedad7a Mon Sep 17 00:00:00 2001
From: Danny Park <dpark@broadinstitute.org>
Date: Tue, 30 Jun 2020 18:05:54 -0400
Subject: [PATCH 1/3] bump viral-core

---
 requirements-modules.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-modules.txt b/requirements-modules.txt
index ee8716e9f..60699affb 100644
--- a/requirements-modules.txt
+++ b/requirements-modules.txt
@@ -1,4 +1,4 @@
-broadinstitute/viral-core=2.1.7
+broadinstitute/viral-core=2.1.8
 broadinstitute/viral-assemble=2.1.4.0
 broadinstitute/viral-classify=2.1.4.0
 broadinstitute/viral-phylo=2.1.4.0

From a00be65050845c09fae55a6fc981dbf5a266fbb6 Mon Sep 17 00:00:00 2001
From: Danny Park <dpark@broadinstitute.org>
Date: Mon, 6 Jul 2020 10:38:06 -0400
Subject: [PATCH 2/3] derisk/simplify Fetch_SRA_to_BAM task to make it 1) more
 fault tolerant for optional metadata fields and 2) rely on shell script/bam
 header for important metaata fields

---
 pipes/WDL/tasks/tasks_ncbi_tools.wdl | 64 ++++++++++++----------------
 1 file changed, 28 insertions(+), 36 deletions(-)

diff --git a/pipes/WDL/tasks/tasks_ncbi_tools.wdl b/pipes/WDL/tasks/tasks_ncbi_tools.wdl
index 935f470d3..a294450db 100644
--- a/pipes/WDL/tasks/tasks_ncbi_tools.wdl
+++ b/pipes/WDL/tasks/tasks_ncbi_tools.wdl
@@ -10,45 +10,37 @@ task Fetch_SRA_to_BAM {
     }
 
     command {
-        set -euxo pipefail
+        # pull reads from SRA and make a fully annotated BAM -- must succeed
+        set -ex
+        /opt/docker/scripts/sra_to_ubam.sh "${SRA_ID}" "${SRA_ID}.bam"
 
-        # pull reads from SRA and make a fully annotated BAM
-        /opt/docker/scripts/sra_to_ubam.sh ${SRA_ID} ${SRA_ID}.bam
+        # pull most metadata from BAM header
+        set +e
+        samtools view -H "${SRA_ID}.bam" | grep ^@RG | head -1 | tr '\t' '\n' > header.txt
+        grep CN header.txt | cut -f 2- -d : | tee OUT_CENTER
+        grep PL header.txt | cut -f 2- -d : | tee OUT_PLATFORM
+        grep SM header.txt | cut -f 2- -d : | tee OUT_BIOSAMPLE
+        grep LB header.txt | cut -f 2- -d : | tee OUT_LIBRARY
+        grep DT header.txt | cut -f 2 -d : | cut -f 1 -d T | tee OUT_RUNDATE
 
-        # pull other metadata from SRA
-        esearch -db sra -q "${SRA_ID}" | efetch -mode json -json > ${SRA_ID}.json
-
-        cat ${SRA_ID}.json | jq -r \
-            '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SUBMISSION.center_name' \
-            | tee OUT_CENTER
-        cat ${SRA_ID}.json | jq -r \
-            '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.EXPERIMENT.PLATFORM | keys[] as $k | "\($k)"' \
-            | tee OUT_PLATFORM
-        cat ${SRA_ID}.json | jq -r \
-            .EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.EXPERIMENT.PLATFORM.$(<OUT_PLATFORM).INSTRUMENT_MODEL \
-            | tee OUT_MODEL
-        cat ${SRA_ID}.json | jq -r \
-            '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SAMPLE.IDENTIFIERS.EXTERNAL_ID|select(.namespace == "BioSample")|.content' \
-            | tee OUT_BIOSAMPLE
-        cat ${SRA_ID}.json | jq -r \
-            .EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.EXPERIMENT.DESIGN.LIBRARY_DESCRIPTOR.LIBRARY_NAME \
-            | tee OUT_LIBRARY
-        cat ${SRA_ID}.json | jq -r \
-            '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.RUN_SET.RUN.SRAFiles.SRAFile[]|select(.supertype == "Original")|.date' \
-            | cut -f 1 -d ' ' \
-            | tee OUT_RUNDATE
-        cat ${SRA_ID}.json | jq -r \
+        # pull other metadata from SRA -- allow for silent failures here!
+        touch OUT_MODEL OUT_COLLECTION_DATE OUT_STRAIN OUT_COLLECTED_BY OUT_GEO_LOC
+        esearch -db sra -q "${SRA_ID}" | efetch -mode json -json > SRA.json
+        jq -r \
+            .EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.EXPERIMENT.PLATFORM."$(<OUT_PLATFORM)".INSTRUMENT_MODEL \
+            SRA.json | tee OUT_MODEL
+        jq -r \
             '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SAMPLE.SAMPLE_ATTRIBUTES.SAMPLE_ATTRIBUTE[]|select(.TAG == "collection_date")|.VALUE' \
-            | tee OUT_COLLECTION_DATE
-        cat ${SRA_ID}.json | jq -r \
+            SRA.json | tee OUT_COLLECTION_DATE
+        jq -r \
             '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SAMPLE.SAMPLE_ATTRIBUTES.SAMPLE_ATTRIBUTE[]|select(.TAG == "strain")|.VALUE' \
-            | tee OUT_STRAIN
-        cat ${SRA_ID}.json | jq -r \
+            SRA.json | tee OUT_STRAIN
+        jq -r \
             '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SAMPLE.SAMPLE_ATTRIBUTES.SAMPLE_ATTRIBUTE[]|select(.TAG == "collected_by")|.VALUE' \
-            | tee OUT_COLLECTED_BY
-        cat ${SRA_ID}.json | jq -r \
+            SRA.json | tee OUT_COLLECTED_BY
+        jq -r \
             '.EXPERIMENT_PACKAGE_SET.EXPERIMENT_PACKAGE.SAMPLE.SAMPLE_ATTRIBUTES.SAMPLE_ATTRIBUTE[]|select(.TAG == "geo_loc_name")|.VALUE' \
-            | tee OUT_GEO_LOC
+            SRA.json | tee OUT_GEO_LOC
     }
 
     output {
@@ -67,10 +59,10 @@ task Fetch_SRA_to_BAM {
     }
 
     runtime {
-        cpu:     4
-        memory:  select_first([machine_mem_gb, 15]) + " GB"
+        cpu:     2
+        memory:  select_first([machine_mem_gb, 6]) + " GB"
         disks:   "local-disk 750 LOCAL"
-        dx_instance_type: "mem2_ssd1_v2_x4"
+        dx_instance_type: "mem2_ssd1_v2_x2"
         docker:  "${docker}"
     }
 }

From f93dc2f96ed054b4a96bcc5eceeb76ac64b6aa7a Mon Sep 17 00:00:00 2001
From: Danny Park <dpark@broadinstitute.org>
Date: Mon, 6 Jul 2020 10:43:03 -0400
Subject: [PATCH 3/3] update ncbi-tools docker image

---
 requirements-modules.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-modules.txt b/requirements-modules.txt
index 60699affb..2c0bc604b 100644
--- a/requirements-modules.txt
+++ b/requirements-modules.txt
@@ -3,6 +3,6 @@ broadinstitute/viral-assemble=2.1.4.0
 broadinstitute/viral-classify=2.1.4.0
 broadinstitute/viral-phylo=2.1.4.0
 broadinstitute/beast-beagle-cuda=1.10.5pre
-broadinstitute/ncbi-tools=2.10.7.0
+broadinstitute/ncbi-tools=2.10.7.1
 nextstrain/base=build-20200629T201240Z
 andersenlabapps/ivar=1.2.2