Skip to content

Commit

Permalink
Make the workflow pluggable (#91)
Browse files Browse the repository at this point in the history 
* make main workflow pluggable

* update svync config handling

* update bedgovcf config handling

* pluggable annotation workflow

* update bam_cnv_calling test

* update all modules

* another wave of module updates

* let the pipeline run again

* fix last modules

* make editorconfig happy

* fix tests
  • Loading branch information
@nvnieuwk
nvnieuwk authored May 29, 2024
1 parent e71fddc commit 11e2958
Show file tree
Hide file tree
Showing 133 changed files with 4,012 additions and 644 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### `Changes`

1. Updated all WisecondorX modules to version 1.2.6 and added the `--seed` argument to `WisecondorX predict`
2. Removed support for the `phenotypes` VEP plugin. Commen VEP plugin support will be added later
3. Made the main workflow pluggable, making it possible to use this pipeline in a meta pipeline
4. Updated all modules to their latest version

### `Fixed`

Expand Down
Empty file added assets/test.npz
View file
Empty file.
92 changes: 45 additions & 47 deletions conf/modules.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
----------------------------------------------------------------------------------------
*/

// When making changes here, make sure to also update the following files: lib/GlobalVariables.groovy
// When making changes here, make sure to also update the following files: workflows/structural.nf
def svCallers = ["delly", "manta", "smoove"] //, "gridss"
def cnvCallers = ["qdnaseq", "wisecondorx"]
def repeatsCallers = ["expansionhunter"]
Expand Down Expand Up @@ -82,7 +82,7 @@ process {
//

if("delly" in callers){
withName: "^.*:BAM_VARIANT_CALLING_DELLY:DELLY_CALL\$" {
withName: "^.*BAM_VARIANT_CALLING_DELLY:DELLY_CALL\$" {
ext.args = {"--svtype ${params.delly_sv_types} --map-qual ${params.delly_map_qual} --min-clique-size ${params.delly_min_clique_size}"}
ext.suffix = "vcf"
ext.prefix = { "${meta.id}.delly" }
Expand All @@ -107,7 +107,7 @@ process {
//

// if("whamg" in callers){
// withName: "^.*:BAM_VARIANT_CALLING_WHAMG:WHAMG\$" {
// withName: "^.*BAM_VARIANT_CALLING_WHAMG:WHAMG\$" {
// ext.prefix = { "${meta.id}.whamg" }
// ext.args = {[
// meta.region ? "-r ${meta.region}" : "" ,
Expand All @@ -129,22 +129,22 @@ process {

if("manta" in callers){

withName: "^.*:BAM_VARIANT_CALLING_MANTA:GAWK\$" {
withName: "^.*BAM_VARIANT_CALLING_MANTA:GAWK\$" {
ext.suffix = "bed"
ext.prefix = "contigs"
ext.args2 = '\'BEGIN {FS="\t"}; \$1 ~ /^chr[1-9XY][1-9]?\$/ {print \$1 FS "0" FS \$2 }\''
}

withName: "^.*:MANTA_.*\$" {
withName: "^.*MANTA_.*\$" {
container = "quay.io/cmgg/manta:1.6.0"
}

withName: "^.*:BAM_VARIANT_CALLING_MANTA:MANTA_GERMLINE\$" {
withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_GERMLINE\$" {
label = "process_high"
cpus = { check_max( 20 * task.attempt, 'cpus' ) }
}

withName: "^.*:BAM_VARIANT_CALLING_MANTA:MANTA_CONVERTINVERSION\$" {
withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_CONVERTINVERSION\$" {
ext.prefix = { "${meta.id}.manta" }
publishDir = [[
enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
Expand All @@ -167,7 +167,7 @@ process {
//

// if("scramble" in callers){
// withName: "^.*:.*:BAM_VARIANT_CALLING_SCRAMBLE:SCRAMBLE_CLUSTERANALYSIS\$" {
// withName: "^.*.*:BAM_VARIANT_CALLING_SCRAMBLE:SCRAMBLE_CLUSTERANALYSIS\$" {
// ext.args = "--eval-meis"
// }
// }
Expand All @@ -177,7 +177,7 @@ process {
//

if("smoove" in callers){
withName: "^.*:BAM_VARIANT_CALLING_SMOOVE:BCFTOOLS_SORT\$" {
withName: "^.*BAM_VARIANT_CALLING_SMOOVE:BCFTOOLS_SORT\$" {
ext.prefix = { "${meta.id}.smoove" }
publishDir = [[
enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
Expand All @@ -194,7 +194,7 @@ process {
]]
}

withName: "^.*:BAM_VARIANT_CALLING_SMOOVE:TABIX_CALLER\$" {
withName: "^.*BAM_VARIANT_CALLING_SMOOVE:TABIX_CALLER\$" {
ext.when = output_callers || (!params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output))
publishDir = [[
enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
Expand All @@ -217,7 +217,7 @@ process {
//

if("gridss" in callers){
withName: "^.*:BAM_VARIANT_CALLING_GRIDSS:GRIDSS_GRIDSS\$" {
withName: "^.*BAM_VARIANT_CALLING_GRIDSS:GRIDSS_GRIDSS\$" {
cpus = { check_max( 8 * task.attempt, 'cpus' ) }
memory = { check_max( 32.GB * task.attempt, 'memory' ) }
ext.prefix = { "${meta.id}.gridss" }
Expand All @@ -235,7 +235,7 @@ process {
]
}

withName: "^.*:BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" {
withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" {
publishDir = [
enabled: output_callers,
overwrite: true,
Expand All @@ -245,13 +245,13 @@ process {
]
}

withName: "^.*:BAM_VARIANT_CALLING_GRIDSS:VIOLA\$" {
withName: "^.*BAM_VARIANT_CALLING_GRIDSS:VIOLA\$" {
time = { check_max( 20.h * task.attempt, 'time') }
ext.prefix = { "${meta.id}.viola" }
ext.args = { meta.read_length ? "--read_length ${meta.read_length}" : ""}
}

withName: "^.*:BAM_VARIANT_CALLING_GRIDSS:BCFTOOLS_SORT\$" {
withName: "^.*BAM_VARIANT_CALLING_GRIDSS:BCFTOOLS_SORT\$" {
ext.prefix = { "${meta.id}.gridss" }
publishDir = [
enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
Expand All @@ -262,7 +262,7 @@ process {
]
}

withName: "^.*:BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" {
withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" {
publishDir = [
enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
overwrite: true,
Expand All @@ -283,7 +283,7 @@ process {
if(foundCnvCallers){

if("qdnaseq" in callers) {
withName: "^.*:BAM_VARIANT_CALLING_QDNASEQ:QDNASEQ.*\$" {
withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:QDNASEQ.*\$" {
cpus = 1
memory = { check_max(50.GB * task.attempt, 'memory') }
ext.prefix = { "${meta.id}.qdnaseq" }
Expand All @@ -302,7 +302,7 @@ process {
]]
}

withName: "^.*:BAM_VARIANT_CALLING_QDNASEQ:GAWK\$" {
withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:GAWK\$" {
ext.prefix = { "${meta.id}.qdnaseq.abberations" }
ext.suffix = "bed"
ext.args = "-F '\\t'"
Expand All @@ -325,7 +325,7 @@ process {
}

if("wisecondorx" in callers) {
withName: "^.*:BAM_VARIANT_CALLING_WISECONDORX:WISECONDORX_PREDICT\$" {
withName: "^.*BAM_VARIANT_CALLING_WISECONDORX:WISECONDORX_PREDICT\$" {
ext.args = "--seed 0 --bed --plot"
memory = { check_max(50.GB * task.attempt, 'memory') }
ext.prefix = { "${meta.id}.wisecondorx" }
Expand All @@ -345,7 +345,7 @@ process {
}
}

withName: "^.*:BAM_CNV_CALLING:[^:]*:BEDGOVCF\$" {
withName: "^.*BAM_CNV_CALLING:[^:]*:BEDGOVCF\$" {
ext.prefix = { "${meta.id}.${meta.caller}" }
ext.args = { meta.caller == "qdnaseq" ? "--skip 1" : "--header" }
publishDir = [[
Expand All @@ -363,7 +363,7 @@ process {
]]
}

withName: "^.*:BAM_CNV_CALLING:[^:]*:TABIX_TABIX\$" {
withName: "^.*BAM_CNV_CALLING:[^:]*:TABIX_TABIX\$" {
publishDir = [[
enabled: !params.annotate && foundCnvCallers.size() == 1 && (count_types == 1 || !params.concat_output),
overwrite: true,
Expand All @@ -388,23 +388,23 @@ process {
*/

if(foundCnvCallers.size() > 1 || foundSvCallers.size() > 1) {
withName: "^.*:VCF_MERGE_CALLERS_JASMINE:JASMINESV\$" {
withName: "^.*VCF_MERGE_CALLERS_JASMINE:JASMINESV\$" {
ext.args = {[
meta.variant_type == "sv" ? "min_support=${params.sv_callers_support}" : "",
meta.variant_type == "cnv" ? "min_support=${params.cnv_callers_support}" : "",
"--allow_intrasample"
].join(' ').trim()}
}

withName: "^.*:VCF_MERGE_CALLERS_JASMINE:FIX_CALLERS\$" {
withName: "^.*VCF_MERGE_CALLERS_JASMINE:FIX_CALLERS\$" {
ext.prefix = { "${meta.id}.callers-corrected" }
}

withName: "^.*:VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" {
withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" {
ext.prefix = { "${meta.id}.reheadered" }
}

withName: "^.*:VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_SORT\$" {
withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_SORT\$" {
ext.prefix = { "${meta.id}.${meta.variant_type}" }
publishDir = [
enabled: !params.annotate && (count_types == 1 || !params.concat_output),
Expand All @@ -415,7 +415,7 @@ process {
]
}

withName: "^.*:VCF_MERGE_CALLERS_JASMINE:TABIX_TABIX\$" {
withName: "^.*VCF_MERGE_CALLERS_JASMINE:TABIX_TABIX\$" {
publishDir = [
enabled: !params.annotate && (count_types == 1 || !params.concat_output),
overwrite: true,
Expand All @@ -442,22 +442,22 @@ process {

if(params.annotate){

withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER\$" {
ext.prefix = {"${meta.id}.filter"}
ext.args = "-e 'GT=\"ref\"' --output-type z"
}

withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ANNOTSV_ANNOTSV\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ANNOTSV_ANNOTSV\$" {
ext.args = "-SVminSize 20"
ext.prefix = {"${meta.id}.annot"}
}

withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_CONCAT\$" {
ext.prefix = params.annotsv_file_name
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_CONCAT\$" {
ext.prefix = "annotsv_annotated"
ext.args = "--output-type z --naive-force"
}

withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ENSEMBLVEP_VEP\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ENSEMBLVEP_VEP\$" {
ext.prefix = {"${meta.id}.vep"}
ext.args = {[
// specify we use VCF files
Expand All @@ -477,18 +477,16 @@ process {
// create stats file
"--stats_file ${prefix}.summary.html",
// proteins
'--domains --biotype --canonical --mane --ccds --protein --polyphen s --sift s',
// plugins
(params.vep_phenotypes) ? "--plugin Phenotypes,file=${params.phenotypes.split('/')[-1]}": ""
'--domains --biotype --canonical --mane --ccds --protein --polyphen s --sift s'
].join(' ').trim()}
}

withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:VCFANNO\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:VCFANNO\$" {
ext.args = "-permissive-overlap -ends"
}

if(!params.annotations_filter) {
withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_ANNOTATED\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_ANNOTATED\$" {
ext.prefix = { "${meta.id}.${meta.variant_type}.annotated" }
publishDir = [
enabled: (count_types == 1 || !params.concat_output),
Expand All @@ -501,7 +499,7 @@ process {
}

if(params.annotations_filter) {
withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER_COMMON\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER_COMMON\$" {
ext.prefix = {"${meta.id}.${meta.variant_type}.annotated"}
ext.args = "${params.annotations_filter} --output-type z"
publishDir = [
Expand All @@ -514,7 +512,7 @@ process {
}


withName: "^.*:VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_FILTER\$" {
withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_FILTER\$" {
publishDir = [
enabled: (count_types == 1 || !params.concat_output),
overwrite: true,
Expand All @@ -535,11 +533,11 @@ process {

if(foundRepeatsCallers){

withName: "^.*:BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:EXPANSIONHUNTER\$" {
withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:EXPANSIONHUNTER\$" {
ext.args = {"--sex ${meta.sex}"}
}

withName: "^.*:BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:BCFTOOLS_ANNOTATE\$" {
withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:BCFTOOLS_ANNOTATE\$" {
ext.args = "-c INFO/REPREF:=INFO/REF --output-type z"
ext.prefix = { "${meta.id}.expansionhunter" }
ext.tabix = true
Expand All @@ -558,7 +556,7 @@ process {
]]
}

withName: "^.*:BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:TABIX_TABIX\$" {
withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:TABIX_TABIX\$" {
publishDir = [[
enabled: (count_types == 1 || !params.concat_output),
overwrite: true,
Expand All @@ -583,7 +581,7 @@ process {

if(count_types > 1 && params.concat_output){

withName: "^.*:VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" {
withName: "^.*VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" {
ext.args = "--output-type z --allow-overlaps"
publishDir = [
enabled: true,
Expand All @@ -594,7 +592,7 @@ process {
]
}

withName: "^.*:VCF_CONCAT_BCFTOOLS:TABIX_TABIX\$" {
withName: "^.*VCF_CONCAT_BCFTOOLS:TABIX_TABIX\$" {
publishDir = [
enabled: true,
overwrite: true,
Expand All @@ -611,19 +609,19 @@ process {
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/

withName: "^.*:VCF_MERGE_FAMILY_JASMINE:JASMINESV\$" {
withName: "^.*VCF_MERGE_FAMILY_JASMINE:JASMINESV\$" {
ext.args = "--output_genotypes"
}

withName: "^.*:VCF_MERGE_FAMILY_JASMINE:FIX_CALLERS\$" {
withName: "^.*VCF_MERGE_FAMILY_JASMINE:FIX_CALLERS\$" {
ext.prefix = { "${meta.id}.callers-corrected" }
}

withName: "^.*:VCF_MERGE_FAMILY_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" {
withName: "^.*VCF_MERGE_FAMILY_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" {
ext.prefix = { "${meta.id}.reheadered" }
}

withName: "^.*:VCF_MERGE_FAMILY_JASMINE:BCFTOOLS_SORT\$" {
withName: "^.*VCF_MERGE_FAMILY_JASMINE:BCFTOOLS_SORT\$" {
ext.args = "--output-type z"
publishDir = [
enabled: true,
Expand All @@ -634,7 +632,7 @@ process {
]
}

withName: "^.*:VCF_MERGE_FAMILY_JASMINE:TABIX_TABIX\$" {
withName: "^.*VCF_MERGE_FAMILY_JASMINE:TABIX_TABIX\$" {
publishDir = [
enabled: true,
overwrite: true,
Expand Down
2 changes: 1 addition & 1 deletion conf/test.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ params {
// Fasta references
fasta = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/reference.fasta"
fai = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/reference.fasta.fai"
bwa = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/bwa.tar.gz"
// bwa = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/bwa.tar.gz"
expansionhunter_catalog = params.test_data["homo_sapiens"]["genome"]["expansionhunter"]
qdnaseq_male = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"]
qdnaseq_female = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"]
Expand Down
13 changes: 0 additions & 13 deletions lib/GlobalVariables.groovy
View file

This file was deleted.

Loading

0 comments on commit 11e2958

Please sign in to comment.