Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

attempt at fixing OpenBLAS-related nproc limit #553

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
+22 −14
Open

attempt at fixing OpenBLAS-related nproc limit #553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
33aa2f1
attempt at fixing OpenBLAS-related nproc limit
tomkinsc Aug 23, 2024
09c6260
bump viral-core 2.3.2->2.3.3; bump viral-classify 2.2.4.2 -> 2.2.5
tomkinsc Aug 23, 2024
c709399
add export OPENBLAS_NUM_THREADS=1 to illumina_demux task
tomkinsc Aug 23, 2024
cf704e8
set export OPENBLAS_NUM_THREADS=1 in fastqc task
tomkinsc Aug 23, 2024
b62f62e
Merge branch 'master' into ct-bugfix-depletion-nproc-limit
tomkinsc Nov 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions pipes/WDL/tasks/tasks_demux.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,8 @@ task illumina_demux {
command <<<
set -ex -o pipefail

export OPENBLAS_NUM_THREADS=1

# find N% memory
mem_in_mb=$(/opt/viral-ngs/source/docker/calc_mem.py mb 85)

Expand Down
2 changes: 1 addition & 1 deletion pipes/WDL/tasks/tasks_megablast.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ task lca_megablast {
Int cpu = 16
Int disk_size_gb = 300

String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}
parameter_meta {
trimmed_fasta: {
Expand Down
16 changes: 8 additions & 8 deletions pipes/WDL/tasks/tasks_metagenomics.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ task kraken2 {
Int? min_base_qual

Int machine_mem_gb = 90
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

parameter_meta {
Expand Down Expand Up @@ -345,7 +345,7 @@ task report_primary_kraken_taxa {
File kraken_summary_report
String focal_taxon = "Viruses"

String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}
String out_basename = basename(kraken_summary_report, '.txt')
Int disk_size = 50
Expand Down Expand Up @@ -396,7 +396,7 @@ task filter_refs_to_found_taxa {
File taxdump_tgz
Int min_read_count = 100

String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}
String ref_basename = basename(taxid_to_ref_accessions_tsv, '.tsv')
String hits_basename = basename(focal_report_tsv, '.tsv')
Expand Down Expand Up @@ -447,7 +447,7 @@ task build_kraken2_db {
Int? zstd_compression_level

Int machine_mem_gb = 100
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

Int disk_size = 750
Expand Down Expand Up @@ -589,7 +589,7 @@ task blastx {
File krona_taxonomy_db_tgz

Int machine_mem_gb = 8
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

parameter_meta {
Expand Down Expand Up @@ -679,7 +679,7 @@ task krona {
Int? magnitude_column

Int machine_mem_gb = 3
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

Int disk_size = 50
Expand Down Expand Up @@ -786,7 +786,7 @@ task filter_bam_to_taxa {
String out_filename_suffix = "filtered"

Int machine_mem_gb = 26
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

String out_basename = basename(classified_bam, ".bam") + "." + out_filename_suffix
Expand Down Expand Up @@ -873,7 +873,7 @@ task kaiju {
File krona_taxonomy_db_tgz # taxonomy/taxonomy.tab

Int machine_mem_gb = 100
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

String input_basename = basename(reads_unmapped_bam, ".bam")
Expand Down
2 changes: 1 addition & 1 deletion pipes/WDL/tasks/tasks_ncbi.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1108,7 +1108,7 @@ task sequence_rename_by_species {
String taxid
File taxdump_tgz

String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}
command <<<
set -e
Expand Down
1 change: 1 addition & 0 deletions pipes/WDL/tasks/tasks_read_utils.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,7 @@ task read_depths {
File aligned_bam

String out_basename = basename(aligned_bam, '.bam')

String docker = "quay.io/broadinstitute/viral-core:2.4.0"
}
Int disk_size = 200
Expand Down
4 changes: 3 additions & 1 deletion pipes/WDL/tasks/tasks_reports.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ task alignment_metrics {
Int max_amplicons=500

Int machine_mem_gb=32

String docker = "quay.io/broadinstitute/viral-core:2.4.0"
}

Expand Down Expand Up @@ -371,6 +372,7 @@ task fastqc {

command {
set -ex -o pipefail
export OPENBLAS_NUM_THREADS=1
reports.py --version | tee VERSION
reports.py fastqc ${reads_bam} ${reads_basename}_fastqc.html --out_zip ${reads_basename}_fastqc.zip
}
Expand Down Expand Up @@ -551,7 +553,7 @@ task aggregate_metagenomics_reports {
String aggregate_taxlevel_focus = "species"
Int aggregate_top_N_hits = 5

String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

parameter_meta {
Expand Down
9 changes: 6 additions & 3 deletions pipes/WDL/tasks/tasks_taxon_filter.wdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ task deplete_taxa {

Int cpu = 8
Int machine_mem_gb = 15
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

parameter_meta {
Expand All @@ -38,6 +38,8 @@ task deplete_taxa {

command <<<
set -ex -o pipefail
export OPENBLAS_NUM_THREADS=1

taxon_filter.py --version | tee VERSION

if [ -z "$TMPDIR" ]; then
Expand Down Expand Up @@ -113,7 +115,7 @@ task filter_to_taxon {
String neg_control_prefixes_space_separated = "neg water NTC"

Int machine_mem_gb = 15
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

# do this in two steps in case the input doesn't actually have "cleaned" in the name
Expand Down Expand Up @@ -172,7 +174,7 @@ task build_lastal_db {
File sequences_fasta

Int machine_mem_gb = 7
String docker = "quay.io/broadinstitute/viral-classify:2.2.4.2"
String docker = "quay.io/broadinstitute/viral-classify:2.2.5"
}

String db_name = basename(sequences_fasta, ".fasta")
Expand Down Expand Up @@ -211,6 +213,7 @@ task merge_one_per_sample {
Boolean rmdup = false

Int machine_mem_gb = 7

String docker = "quay.io/broadinstitute/viral-core:2.4.0"
}

Expand Down
Loading