diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27009439..014b1798 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: strategy: matrix: NXF_VER: - - "23.10.0" + - "24.04.0" - "latest-everything" test: - "pipeline_sv" @@ -70,7 +70,7 @@ jobs: - name: Install nf-test run: | - conda install -c bioconda nf-test==0.8.4 + conda install -c bioconda nf-test - name: Run pipeline with test data run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index 22146f10..05d85188 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changes` +1. Updated the pipeline to be compatible with future nextflow updates +2. Removed the `max_cpus`, `max_memory` and `max_time` parameter in favor of the new built-in `resourceLimits` +3. Replaced `nf-validation` with `nf-schema` + ### `Fixed` 1. Fail the pipeline when the sex determination failed. This will now prompt the user to add the sex to the samplesheet so the pipeline doesn't do any wrong assumptions diff --git a/README.md b/README.md index 59c199a9..83bf757e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![GitHub Actions Linting Status](https://github.com/nf-cmgg/structural/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-cmgg/structural/actions/workflows/linting.yml) [![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.10.0-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A524.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/assets/schema_input.json b/assets/schema_input.json index 62dec9ff..d0f91cd4 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/structural/master/assets/schema_input.json", "title": "nf-cmgg/structural pipeline - params.input schema", "description": "Schema for the file provided with params.input", diff --git a/conf/base.config b/conf/base.config index cd13234a..086f42a8 100644 --- a/conf/base.config +++ b/conf/base.config @@ -10,47 +10,40 @@ process { - // TODO nf-core: Check the defaults for all processes - cpus = { check_max( 1 * task.attempt, 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 * task.attempt } + memory = { 8.GB * task.attempt } + time = { 4.h * task.attempt } errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' } maxRetries = 1 maxErrors = '-1' // Process-specific resource requirements - // NOTE - Please try and re-use the labels below as much as possible. - // These labels are used and recognised by default in DSL2 files hosted on nf-core/modules. - // If possible, it would be nice to keep the same label naming convention when - // adding in your local modules too. - // TODO nf-core: Customise requirements for specific processes. - // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors withLabel:process_single { - cpus = { check_max( 1 , 'cpus' ) } - memory = { check_max( 6.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 1 } + memory = { 8.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_low { - cpus = { check_max( 2 * task.attempt, 'cpus' ) } - memory = { check_max( 12.GB * task.attempt, 'memory' ) } - time = { check_max( 4.h * task.attempt, 'time' ) } + cpus = { 2 * task.attempt } + memory = { 16.GB * task.attempt } + time = { 4.h * task.attempt } } withLabel:process_medium { - cpus = { check_max( 6 * task.attempt, 'cpus' ) } - memory = { check_max( 36.GB * task.attempt, 'memory' ) } - time = { check_max( 8.h * task.attempt, 'time' ) } + cpus = { 4 * task.attempt } + memory = { 32.GB * task.attempt } + time = { 8.h * task.attempt } } withLabel:process_high { - cpus = { check_max( 12 * task.attempt, 'cpus' ) } - memory = { check_max( 72.GB * task.attempt, 'memory' ) } - time = { check_max( 16.h * task.attempt, 'time' ) } + cpus = { 8 * task.attempt } + memory = { 64.GB * task.attempt } + time = { 16.h * task.attempt } } withLabel:process_long { - time = { check_max( 20.h * task.attempt, 'time' ) } + time = { 20.h * task.attempt } } withLabel:process_high_memory { - memory = { check_max( 200.GB * task.attempt, 'memory' ) } + memory = { 200.GB * task.attempt } } withLabel:error_ignore { errorStrategy = 'ignore' @@ -59,7 +52,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/conf/empty.config b/conf/empty.config new file mode 100644 index 00000000..e69de29b diff --git a/conf/empty_genomes.config b/conf/empty_genomes.config new file mode 100644 index 00000000..2e919fc0 --- /dev/null +++ b/conf/empty_genomes.config @@ -0,0 +1 @@ +params.genomes = [:] \ No newline at end of file diff --git a/conf/empty_test_data.config b/conf/empty_test_data.config new file mode 100644 index 00000000..58544ddd --- /dev/null +++ b/conf/empty_test_data.config @@ -0,0 +1 @@ +params.test_data = [:] \ No newline at end of file diff --git a/conf/modules.config b/conf/modules.config index f6c31019..65b11e92 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -10,6 +10,7 @@ ---------------------------------------------------------------------------------------- */ +// TODO remove this once the new workflow output definitions have been fully integrated in Nextflow // When making changes here, make sure to also update the following files: workflows/structural.nf def svCallers = ["delly", "manta", "smoove"] //, "gridss" def cnvCallers = ["qdnaseq", "wisecondorx"] @@ -49,16 +50,12 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(params.annotate && callers.intersect(annotationCallers)) { - withName: ENSEMBLVEP_DOWNLOAD { - ext.args = "--AUTO c --CONVERT --NO_BIOPERL --NO_TEST --NO_UPDATE" - } + withName: ENSEMBLVEP_DOWNLOAD { + ext.args = "--AUTO c --CONVERT --NO_BIOPERL --NO_TEST --NO_UPDATE" } - if("gridss" in callers) { - withName: BWA_INDEX { - ext.prefix = { "${fasta}" } - } + withName: BWA_INDEX { + ext.prefix = { "${fasta}" } } /* @@ -67,318 +64,297 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(foundSvCallers){ - - // - // Svync (used for all callers) - // - - withName: SVYNC { - ext.prefix = { "${meta.id}.svync.${meta.caller}"} - } - - // - // Delly - // - - if("delly" in callers){ - withName: "^.*BAM_VARIANT_CALLING_DELLY:DELLY_CALL\$" { - ext.args = {"--svtype ${params.delly_sv_types} --map-qual ${params.delly_map_qual} --min-clique-size ${params.delly_min_clique_size}"} - ext.suffix = "vcf" - ext.prefix = { "${meta.id}.delly" } - publishDir = [[ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename.replace("delly", "sv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/delly" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } - } - - // - // Whamg - // - - // if("whamg" in callers){ - // withName: "^.*BAM_VARIANT_CALLING_WHAMG:WHAMG\$" { - // ext.prefix = { "${meta.id}.whamg" } - // ext.args = {[ - // meta.region ? "-r ${meta.region}" : "" , - // "-z" - // ].join(' ').trim()} - // publishDir = [ - // enabled: output_callers, - // overwrite: true, - // path: { "${params.outdir}/${meta.id}/whamg" }, - // mode: params.publish_dir_mode, - // saveAs: { filename -> filename != "versions.yml" ? filename : null } - // ] - // } - // } - - // - // Manta - // - - if("manta" in callers){ - - withName: "^.*BAM_VARIANT_CALLING_MANTA:GAWK\$" { - ext.suffix = "bed" - ext.prefix = "contigs" - ext.args2 = '\'BEGIN {FS="\t"}; \$1 ~ /^chr[1-9XY][1-9]?\$/ {print \$1 FS "0" FS \$2 }\'' - } + // + // Svync (used for all callers) + // - withName: "^.*MANTA_.*\$" { - container = "quay.io/cmgg/manta:1.6.0" - } + withName: SVYNC { + ext.prefix = { "${meta.id}.svync.${meta.caller}"} + } - withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_GERMLINE\$" { - label = "process_high" - cpus = { check_max( 20 * task.attempt, 'cpus' ) } - } + // + // Delly + // - withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_CONVERTINVERSION\$" { - ext.prefix = { "${meta.id}.manta" } - publishDir = [[ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename.replace("manta", "sv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/manta" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } - } - - // - // Scramble - // - - // if("scramble" in callers){ - // withName: "^.*.*:BAM_VARIANT_CALLING_SCRAMBLE:SCRAMBLE_CLUSTERANALYSIS\$" { - // ext.args = "--eval-meis" - // } - // } - - // - // Smoove - // - - if("smoove" in callers){ - withName: "^.*BAM_VARIANT_CALLING_SMOOVE:BCFTOOLS_SORT\$" { - ext.prefix = { "${meta.id}.smoove" } - publishDir = [[ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename.replace("smoove", "sv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/smoove" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } + withName: "^.*BAM_VARIANT_CALLING_DELLY:DELLY_CALL\$" { + ext.args = {"--svtype ${params.delly_sv_types} --map-qual ${params.delly_map_qual} --min-clique-size ${params.delly_min_clique_size}"} + ext.suffix = "vcf" + ext.prefix = { "${meta.id}.delly" } + publishDir = [[ + enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename.replace("delly", "sv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/delly" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } - withName: "^.*BAM_VARIANT_CALLING_SMOOVE:TABIX_CALLER\$" { - ext.when = output_callers || (!params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output)) - publishDir = [[ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename.replace("smoove", "sv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/smoove" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ]] - } - } - - // - // Gridss - // - - if("gridss" in callers){ - withName: "^.*BAM_VARIANT_CALLING_GRIDSS:GRIDSS_GRIDSS\$" { - cpus = { check_max( 8 * task.attempt, 'cpus' ) } - memory = { check_max( 32.GB * task.attempt, 'memory' ) } - ext.prefix = { "${meta.id}.gridss" } - ext.args = { [ - "--steps preprocess,assemble,call", - "--otherjvmheap ${task.memory.toGiga() > 10 ? 4 : 2}g", - "--jvmheap ${task.memory.toGiga() > 10 ? task.memory.toGiga() - 8 : 2}g" - ].join(" ") } - publishDir = [ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/gridss" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename ==~ /^.*.vcf.gz$/ ? filename : null } - ] - } + // + // Whamg + // + + // if("whamg" in callers){ + // withName: "^.*BAM_VARIANT_CALLING_WHAMG:WHAMG\$" { + // ext.prefix = { "${meta.id}.whamg" } + // ext.args = {[ + // meta.region ? "-r ${meta.region}" : "" , + // "-z" + // ].join(' ').trim()} + // publishDir = [ + // enabled: output_callers, + // overwrite: true, + // path: { "${params.outdir}/${meta.id}/whamg" }, + // mode: params.publish_dir_mode, + // saveAs: { filename -> filename != "versions.yml" ? filename : null } + // ] + // } + // } + + // + // Manta + // + + withName: "^.*BAM_VARIANT_CALLING_MANTA:GAWK\$" { + ext.suffix = "bed" + ext.prefix = "contigs" + ext.args2 = '\'BEGIN {FS="\t"}; \$1 ~ /^chr[1-9XY][1-9]?\$/ {print \$1 FS "0" FS \$2 }\'' + } - withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" { - publishDir = [ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/gridss" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ] - } + withName: "^.*MANTA_.*\$" { + container = "quay.io/cmgg/manta:1.6.0" + } - withName: "^.*BAM_VARIANT_CALLING_GRIDSS:VIOLA\$" { - time = { check_max( 20.h * task.attempt, 'time') } - ext.prefix = { "${meta.id}.viola" } - ext.args = { meta.read_length ? "--read_length ${meta.read_length}" : ""} - } + withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_GERMLINE\$" { + label = "process_high" + cpus = { 20 * task.attempt } + } - withName: "^.*BAM_VARIANT_CALLING_GRIDSS:BCFTOOLS_SORT\$" { - ext.prefix = { "${meta.id}.gridss" } - publishDir = [ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename.replace("gridss", "sv") : null } - ] - } + withName: "^.*BAM_VARIANT_CALLING_MANTA:MANTA_CONVERTINVERSION\$" { + ext.prefix = { "${meta.id}.manta" } + publishDir = [[ + enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename.replace("manta", "sv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/manta" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } - withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" { - publishDir = [ - enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename.replace("gridss", "sv") : null } - ] - } - } + // + // Scramble + // + + // if("scramble" in callers){ + // withName: "^.*.*:BAM_VARIANT_CALLING_SCRAMBLE:SCRAMBLE_CLUSTERANALYSIS\$" { + // ext.args = "--eval-meis" + // } + // } + + // + // Smoove + // + + withName: "^.*BAM_VARIANT_CALLING_SMOOVE:BCFTOOLS_SORT\$" { + ext.prefix = { "${meta.id}.smoove" } + publishDir = [[ + enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename.replace("smoove", "sv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/smoove" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] } + withName: "^.*BAM_VARIANT_CALLING_SMOOVE:TABIX_CALLER\$" { + ext.when = output_callers || (!params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output)) + publishDir = [[ + enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename.replace("smoove", "sv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/smoove" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ]] + } + + // + // Gridss + // + + // withName: "^.*BAM_VARIANT_CALLING_GRIDSS:GRIDSS_GRIDSS\$" { + // cpus = { 8 * task.attempt } + // memory = { 32.GB * task.attempt } + // ext.prefix = { "${meta.id}.gridss" } + // ext.args = { [ + // "--steps preprocess,assemble,call", + // "--otherjvmheap ${task.memory.toGiga() > 10 ? 4 : 2}g", + // "--jvmheap ${task.memory.toGiga() > 10 ? task.memory.toGiga() - 8 : 2}g" + // ].join(" ") } + // publishDir = [ + // enabled: output_callers, + // overwrite: true, + // path: { "${params.outdir}/${meta.id}/gridss" }, + // mode: params.publish_dir_mode, + // saveAs: { filename -> filename ==~ /^.*.vcf.gz$/ ? filename : null } + // ] + // } + + // withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" { + // publishDir = [ + // enabled: output_callers, + // overwrite: true, + // path: { "${params.outdir}/${meta.id}/gridss" }, + // mode: params.publish_dir_mode, + // saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + // ] + // } + + // withName: "^.*BAM_VARIANT_CALLING_GRIDSS:VIOLA\$" { + // time = { 20.h * task.attempt } + // ext.prefix = { "${meta.id}.viola" } + // ext.args = { meta.read_length ? "--read_length ${meta.read_length}" : ""} + // } + + // withName: "^.*BAM_VARIANT_CALLING_GRIDSS:BCFTOOLS_SORT\$" { + // ext.prefix = { "${meta.id}.gridss" } + // publishDir = [ + // enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + // overwrite: true, + // path: { "${params.outdir}/${meta.id}" }, + // mode: params.publish_dir_mode, + // saveAs: { filename -> filename != "versions.yml" ? filename.replace("gridss", "sv") : null } + // ] + // } + + // withName: "^.*BAM_VARIANT_CALLING_GRIDSS:TABIX_TABIX\$" { + // publishDir = [ + // enabled: !params.annotate && foundSvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + // overwrite: true, + // path: { "${params.outdir}/${meta.id}" }, + // mode: params.publish_dir_mode, + // saveAs: { filename -> filename.endsWith(".tbi") ? filename.replace("gridss", "sv") : null } + // ] + // } + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ COPY NUMBER VARIATION ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(foundCnvCallers){ - - if("qdnaseq" in callers) { - withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:QDNASEQ.*\$" { - cpus = 1 - memory = { check_max(50.GB * task.attempt, 'memory') } - ext.prefix = { "${meta.id}.qdnaseq" } - publishDir = [[ - enabled: !output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/qdnaseq" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } + withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:QDNASEQ.*\$" { + cpus = 1 + memory = { 50.GB * task.attempt } + ext.prefix = { "${meta.id}.qdnaseq" } + publishDir = [[ + enabled: !output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/qdnaseq" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } - withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:GAWK\$" { - ext.prefix = { "${meta.id}.qdnaseq.abberations" } - ext.suffix = "bed" - ext.args = "-F '\\t'" - ext.args2 = "'{ if (\$5 >= ${params.qdnaseq_cnv_ratio} || \$5 <= -${params.qdnaseq_cnv_ratio} || \$5 == \"Inf\") { print } }'" - publishDir = [[ - enabled: true, - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/qdnaseq" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } + withName: "^.*BAM_VARIANT_CALLING_QDNASEQ:GAWK\$" { + ext.prefix = { "${meta.id}.qdnaseq.abberations" } + ext.suffix = "bed" + ext.args = "-F '\\t'" + ext.args2 = "'{ if (\$5 >= ${params.qdnaseq_cnv_ratio} || \$5 <= -${params.qdnaseq_cnv_ratio} || \$5 == \"Inf\") { print } }'" + publishDir = [[ + enabled: true, + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/qdnaseq" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } - } - - if("wisecondorx" in callers) { - withName: "^.*BAM_VARIANT_CALLING_WISECONDORX:WISECONDORX_PREDICT\$" { - ext.args = "--seed 0 --bed --plot" - memory = { check_max(50.GB * task.attempt, 'memory') } - ext.prefix = { "${meta.id}.wisecondorx" } - publishDir = [[ - enabled: !output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/wisecondorx" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } - } - - withName: "^.*BAM_CNV_CALLING:[^:]*:BEDGOVCF\$" { - ext.prefix = { "${meta.id}.${meta.caller}" } - ext.args = { meta.caller == "qdnaseq" ? "--skip 1" : "--header" } - publishDir = [[ - enabled: !params.annotate && foundCnvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith("vcf.gz") ? filename.replace("${meta.caller}", "cnv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/${meta.caller}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } - - withName: "^.*BAM_CNV_CALLING:[^:]*:TABIX_TABIX\$" { - publishDir = [[ - enabled: !params.annotate && foundCnvCallers.size() == 1 && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith("vcf.gz.tbi") ? filename.replace("${meta.caller}", "cnv") : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/${meta.caller}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ]] - } + withName: "^.*BAM_VARIANT_CALLING_WISECONDORX:WISECONDORX_PREDICT\$" { + ext.args = "--seed 0 --bed --plot" + memory = { 50.GB * task.attempt } + ext.prefix = { "${meta.id}.wisecondorx" } + publishDir = [[ + enabled: !output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/wisecondorx" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } + + withName: BEDGOVCF { + ext.prefix = { "${meta.id}.${meta.caller}" } + ext.args = { meta.caller == "qdnaseq" ? "--skip 1" : "--header" } + publishDir = [[ + enabled: !params.annotate && foundCnvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith("vcf.gz") ? filename.replace("${meta.caller}", "cnv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/${meta.caller}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } + withName: "^.*(WISECONDORX|QDNASEQ):TABIX_TABIX\$" { + publishDir = [[ + enabled: !params.annotate && foundCnvCallers.size() == 1 && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith("vcf.gz.tbi") ? filename.replace("${meta.caller}", "cnv") : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/${meta.caller}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ]] } /* @@ -387,51 +363,49 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(foundCnvCallers.size() > 1 || foundSvCallers.size() > 1) { - withName: "^.*VCF_MERGE_CALLERS_JASMINE:JASMINESV\$" { - ext.args = {[ - meta.variant_type == "sv" ? "min_support=${params.sv_callers_support}" : "", - meta.variant_type == "cnv" ? "min_support=${params.cnv_callers_support}" : "", - "--allow_intrasample" - ].join(' ').trim()} - } - - withName: "^.*VCF_MERGE_CALLERS_JASMINE:FIX_CALLERS\$" { - ext.prefix = { "${meta.id}.callers-corrected" } - } - - withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" { - ext.prefix = { "${meta.id}.reheadered" } - } - - withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_SORT\$" { - ext.prefix = { "${meta.id}.${meta.variant_type}" } - publishDir = [ - enabled: !params.annotate && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ] - } - - withName: "^.*VCF_MERGE_CALLERS_JASMINE:TABIX_TABIX\$" { - publishDir = [ - enabled: !params.annotate && (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> - if(meta.variant_type == "sv" && foundSvCallers.size() == 1) { - return null - } - else if(meta.variant_type == "cnv" && foundSvCallers.size() == 1) { - return null - } - return filename.endsWith(".tbi") ? filename : null + withName: "^.*VCF_MERGE_CALLERS_JASMINE:JASMINESV\$" { + ext.args = {[ + meta.variant_type == "sv" ? "min_support=${params.sv_callers_support}" : "", + meta.variant_type == "cnv" ? "min_support=${params.cnv_callers_support}" : "", + "--allow_intrasample" + ].join(' ').trim()} + } + + withName: "^.*VCF_MERGE_CALLERS_JASMINE:FIX_CALLERS\$" { + ext.prefix = { "${meta.id}.callers-corrected" } + } + + withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_CONSENSUS_REHEADER\$" { + ext.prefix = { "${meta.id}.reheadered" } + } + + withName: "^.*VCF_MERGE_CALLERS_JASMINE:BCFTOOLS_SORT\$" { + ext.prefix = { "${meta.id}.${meta.variant_type}" } + publishDir = [ + enabled: !params.annotate && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ] + } + + withName: "^.*VCF_MERGE_CALLERS_JASMINE:TABIX_TABIX\$" { + publishDir = [ + enabled: !params.annotate && (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> + if(meta.variant_type == "sv" && foundSvCallers.size() == 1) { + return null } - ] - } + else if(meta.variant_type == "cnv" && foundSvCallers.size() == 1) { + return null + } + return filename.endsWith(".tbi") ? filename : null + } + ] } /* @@ -440,93 +414,85 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(params.annotate){ - - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER\$" { - ext.prefix = {"${meta.id}.filter"} - ext.args = "-e 'GT=\"ref\"' --output-type z" - } - - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ANNOTSV_ANNOTSV\$" { - ext.args = {[ - "-SVminSize 20", - "-vcf 1", - meta.hpo ? "-hpo ${meta.hpo}" : "" - ].join(" ")} - ext.prefix = {"${meta.id}.annot"} - } - - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_CONCAT\$" { - ext.prefix = "annotsv_annotated" - ext.args = "--output-type z --naive-force" - } - - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ENSEMBLVEP_VEP\$" { - ext.prefix = {"${meta.id}.vep"} - ext.args = {[ - // specify we use VCF files - '--format vcf', - // don't contact external db - '--offline', - // increase buffer_size to speed up analysis - '--buffer_size 100000', - // output format options - '--vcf --compress_output bgzip --force_overwrite', - // co-located variant info - '--check_existing', - // specific options for structural variants - '--overlaps', // TODO define the best configuration for --max_sv_size, --batch_size, --check_svs and --database - // annotations - '--regulatory --pubmed --symbol --hgvsg --hgvs --af_gnomadg --sift s', - // create stats file - "--stats_file ${prefix}.summary.html", - // proteins - '--domains --biotype --canonical --mane --ccds --protein --polyphen s --sift s' - ].join(' ').trim()} - } - - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:VCFANNO\$" { - ext.args = "-permissive-overlap -ends" - } - - if(!params.annotations_filter) { - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_ANNOTATED\$" { - ext.prefix = { "${meta.id}.${meta.variant_type}.annotated" } - publishDir = [ - enabled: (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" && !filename.endsWith(".csi") ? filename : null } - ] - } - } - - if(params.annotations_filter) { - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER_COMMON\$" { - ext.prefix = {"${meta.id}.${meta.variant_type}.annotated"} - ext.args = "${params.annotations_filter} --output-type z" - publishDir = [ - enabled: (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ] - } + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER\$" { + ext.prefix = {"${meta.id}.filter"} + ext.args = "-e 'GT=\"ref\"' --output-type z" + } + + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ANNOTSV_ANNOTSV\$" { + ext.args = {[ + "-SVminSize 20", + "-vcf 1", + meta.hpo ? "-hpo ${meta.hpo}" : "" + ].join(" ")} + ext.prefix = {"${meta.id}.annot"} + } + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_CONCAT\$" { + ext.prefix = "annotsv_annotated" + ext.args = "--output-type z --naive-force" + } + + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:ENSEMBLVEP_VEP\$" { + ext.prefix = {"${meta.id}.vep"} + ext.args = {[ + // specify we use VCF files + '--format vcf', + // don't contact external db + '--offline', + // increase buffer_size to speed up analysis + '--buffer_size 100000', + // output format options + '--vcf --compress_output bgzip --force_overwrite', + // co-located variant info + '--check_existing', + // specific options for structural variants + '--overlaps', // TODO define the best configuration for --max_sv_size, --batch_size, --check_svs and --database + // annotations + '--regulatory --pubmed --symbol --hgvsg --hgvs --af_gnomadg --sift s', + // create stats file + "--stats_file ${prefix}.summary.html", + // proteins + '--domains --biotype --canonical --mane --ccds --protein --polyphen s --sift s' + ].join(' ').trim()} + } + + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:VCFANNO\$" { + ext.args = "-permissive-overlap -ends" + } + + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_ANNOTATED\$" { + ext.prefix = { "${meta.id}.${meta.variant_type}.annotated" } + publishDir = [ + enabled: (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" && !filename.endsWith(".csi") ? filename : null } + ] + } + + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:BCFTOOLS_FILTER_COMMON\$" { + ext.prefix = {"${meta.id}.${meta.variant_type}.annotated"} + ext.args = "${params.annotations_filter} --output-type z" + publishDir = [ + enabled: (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ] + } - withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_FILTER\$" { - publishDir = [ - enabled: (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ] - } - } + withName: "^.*VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO:TABIX_FILTER\$" { + publishDir = [ + enabled: (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ] } /* @@ -535,46 +501,43 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(foundRepeatsCallers){ - - withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:EXPANSIONHUNTER\$" { - ext.args = {"--sex ${meta.sex}"} - } - - withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:BCFTOOLS_ANNOTATE\$" { - ext.args = "-c INFO/REPREF:=INFO/REF --output-type z" - ext.prefix = { "${meta.id}.expansionhunter" } - ext.tabix = true - publishDir = [[ - enabled: (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/expansionhunter" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ]] - } - - withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:TABIX_TABIX\$" { - publishDir = [[ - enabled: (count_types == 1 || !params.concat_output), - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ],[ - enabled: output_callers, - overwrite: true, - path: { "${params.outdir}/${meta.id}/expansionhunter" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ]] - } + withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:EXPANSIONHUNTER\$" { + ext.args = {"--sex ${meta.sex}"} + } + + withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:BCFTOOLS_ANNOTATE\$" { + ext.args = "-c INFO/REPREF:=INFO/REF --output-type z" + ext.prefix = { "${meta.id}.expansionhunter" } + ext.tabix = true + publishDir = [[ + enabled: (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/expansionhunter" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ]] + } + + withName: "^.*BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER:TABIX_TABIX\$" { + publishDir = [[ + enabled: (count_types == 1 || !params.concat_output), + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ],[ + enabled: output_callers, + overwrite: true, + path: { "${params.outdir}/${meta.id}/expansionhunter" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ]] } /* @@ -583,28 +546,25 @@ process { ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ - if(count_types > 1 && params.concat_output){ - - withName: "^.*VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" { - ext.args = "--output-type z --allow-overlaps" - publishDir = [ - enabled: true, - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename != "versions.yml" ? filename : null } - ] - } - - withName: "^.*VCF_CONCAT_BCFTOOLS:TABIX_TABIX\$" { - publishDir = [ - enabled: true, - overwrite: true, - path: { "${params.outdir}/${meta.id}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } - ] - } + withName: "^.*VCF_CONCAT_BCFTOOLS:BCFTOOLS_CONCAT\$" { + ext.args = "--output-type z --allow-overlaps" + publishDir = [ + enabled: true, + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename != "versions.yml" ? filename : null } + ] + } + + withName: "^.*VCF_CONCAT_BCFTOOLS:TABIX_TABIX\$" { + publishDir = [ + enabled: true, + overwrite: true, + path: { "${params.outdir}/${meta.id}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.endsWith(".tbi") ? filename : null } + ] } /* diff --git a/conf/test.config b/conf/test.config index 9c5666fd..60ed46b4 100644 --- a/conf/test.config +++ b/conf/test.config @@ -10,15 +10,13 @@ ---------------------------------------------------------------------------------------- */ +// Include test data config +includeConfig !System.getenv('NXF_OFFLINE') ? 'https://raw.githubusercontent.com/nf-cmgg/test-datasets/main/nf-cmgg-testdata.config' : "conf/empty_test_data.config" + params { config_profile_name = 'Test profile' config_profile_description = 'Minimal test dataset to check pipeline function' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - // Input data // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets // TODO nf-core: Give any required params for the test so that command line flags are not needed @@ -29,8 +27,8 @@ params { fai = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/reference.fasta.fai" // bwa = "https://github.com/nf-cmgg/test-datasets/raw/main/data/genomics/homo_sapiens/genome/seq/SVcontrol/bwa.tar.gz" expansionhunter_catalog = params.test_data["homo_sapiens"]["genome"]["expansionhunter"] - qdnaseq_male = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"] - qdnaseq_female = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"] + qdnaseq_male = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"] + qdnaseq_female = params.test_data["homo_sapiens"]["genome"]["genome_qdnaseq"] igenomes_ignore = true genomes_ignore = true genome = 'GRCh38' @@ -43,3 +41,12 @@ params { callers = "delly,manta,smoove,expansionhunter,qdnaseq" //,gridss output_callers = true } + +process { + // Limit resources so that this can run on GitHub Actions + resourceLimits = [ + cpus : 2, + memory: 6.GB, + time : 6.h + ] +} diff --git a/main.nf b/main.nf index 8b24bb3b..a4fad079 100644 --- a/main.nf +++ b/main.nf @@ -7,7 +7,7 @@ ---------------------------------------------------------------------------------------- */ -include { validateParameters; paramsHelp; paramsSummaryLog } from 'plugin/nf-validation' +include { validateParameters; paramsHelp; paramsSummaryLog } from 'plugin/nf-schema' nextflow.enable.dsl = 2 @@ -18,28 +18,6 @@ nextflow.enable.dsl = 2 */ include { getGenomeAttribute } from './subworkflows/local/utils_nfcore_structural_pipeline' -params.fasta = getGenomeAttribute('fasta') -params.fai = getGenomeAttribute('fai') -params.vep_cache = getGenomeAttribute('vep_cache') -// params.bwa = getGenomeAttribute('bwa') -params.annotsv_annotations = getGenomeAttribute('annotsv_annotations') -params.expansionhunter_catalog = getGenomeAttribute('expansionhunter_catalog') -params.qdnaseq_male = getGenomeAttribute("qdnaseq_male_${params.qdnaseq_bin_size.toInteger() / 1000}kbp".toString()) -params.qdnaseq_female = getGenomeAttribute("qdnaseq_female_${params.qdnaseq_bin_size.toInteger() / 1000}kbp".toString()) -params.wisecondorx_reference = getGenomeAttribute('wisecondorx_reference') - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CONFIG FILES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS @@ -62,10 +40,23 @@ include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_stru workflow NFCMGG_STRUCTURAL { take: - samplesheet // channel: samplesheet read in from --input + samplesheet // channel: samplesheet read in from --input + pipeline_params // map: The parameters needed for the pipeline to run main: + /* + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + CONFIG FILES + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + */ + + ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = pipeline_params.multiqc_config ? Channel.fromPath( pipeline_params.multiqc_config, checkIfExists: true ) : Channel.empty() + ch_multiqc_logo = pipeline_params.multiqc_logo ? Channel.fromPath( pipeline_params.multiqc_logo, checkIfExists: true ) : Channel.empty() + ch_multiqc_custom_methods_description = pipeline_params.multiqc_methods_description ? file(pipeline_params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + + // // WORKFLOW: Run pipeline // @@ -78,39 +69,39 @@ workflow NFCMGG_STRUCTURAL { ch_multiqc_custom_methods_description, // file inputs - params.fasta, - params.fai, - params.expansionhunter_catalog ?: "https://github.com/Illumina/ExpansionHunter/raw/master/variant_catalog/grch38/variant_catalog.json", - params.qdnaseq_female, - params.qdnaseq_male, - params.wisecondorx_reference, - params.vep_cache, - params.annotsv_annotations, - params.annotsv_candidate_genes, - params.annotsv_gene_transcripts, - params.vcfanno_lua, - params.vcfanno_resources, - params.vcfanno_toml, - params.blacklist, - params.manta_config ?: "${projectDir}/assets/manta_config.ini", + pipeline_params.fasta, + pipeline_params.fai, + pipeline_params.expansionhunter_catalog ?: "https://github.com/Illumina/ExpansionHunter/raw/master/variant_catalog/grch38/variant_catalog.json", + pipeline_params.qdnaseq_female, + pipeline_params.qdnaseq_male, + pipeline_params.wisecondorx_reference, + pipeline_params.vep_cache, + pipeline_params.annotsv_annotations, + pipeline_params.annotsv_candidate_genes, + pipeline_params.annotsv_gene_transcripts, + pipeline_params.vcfanno_lua, + pipeline_params.vcfanno_resources, + pipeline_params.vcfanno_toml, + pipeline_params.blacklist, + pipeline_params.manta_config ?: "${projectDir}/assets/manta_config.ini", "${projectDir}/assets/svync", "${projectDir}/assets/bedgovcf", "${projectDir}/assets/vcfanno", // boolean inputs - params.annotate, - params.concat_output, + pipeline_params.annotate, + pipeline_params.concat_output, // value inputs - params.callers, - params.sv_callers_support, - params.cnv_callers_support, - params.genome, - params.species, - params.vep_assembly, - params.vep_cache_version, - params.annotations_filter, - params.outdir + pipeline_params.callers, + pipeline_params.sv_callers_support, + pipeline_params.cnv_callers_support, + pipeline_params.genome, + pipeline_params.species, + pipeline_params.vep_assembly, + pipeline_params.vep_cache_version, + pipeline_params.annotations_filter, + pipeline_params.outdir ) emit: @@ -127,6 +118,19 @@ workflow { main: + // + // Initialize genome parameters + // + params.fasta = getGenomeAttribute('fasta', params.genome, params.genomes) + params.fai = getGenomeAttribute('fai', params.genome, params.genomes) + params.vep_cache = getGenomeAttribute('vep_cache', params.genome, params.genomes) + // params.bwa = getGenomeAttribute('bwa', params.genome, params.genomes) + params.annotsv_annotations = getGenomeAttribute('annotsv_annotations', params.genome, params.genomes) + params.expansionhunter_catalog = getGenomeAttribute('expansionhunter_catalog', params.genome, params.genomes) + params.qdnaseq_male = getGenomeAttribute("qdnaseq_male_${params.qdnaseq_bin_size.toInteger() / 1000}kbp".toString(), params.genome, params.genomes) + params.qdnaseq_female = getGenomeAttribute("qdnaseq_female_${params.qdnaseq_bin_size.toInteger() / 1000}kbp".toString(), params.genome, params.genomes) + params.wisecondorx_reference = getGenomeAttribute('wisecondorx_reference', params.genome, params.genomes) + // // SUBWORKFLOW: Run initialisation tasks // @@ -137,14 +141,17 @@ workflow { params.monochrome_logs, args, params.outdir, - params.input + params.input, + params.genome, + params.genomes ) // // WORKFLOW: Run main workflow // NFCMGG_STRUCTURAL ( - PIPELINE_INITIALISATION.out.samplesheet + PIPELINE_INITIALISATION.out.samplesheet, + params ) // diff --git a/modules.json b/modules.json index d15be54c..dbbf78e3 100644 --- a/modules.json +++ b/modules.json @@ -8,163 +8,225 @@ "annotsv/annotsv": { "branch": "master", "git_sha": "a94ad45fa5b350961c374c46f79bc86cd5853353", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "annotsv/installannotations": { "branch": "master", "git_sha": "a94ad45fa5b350961c374c46f79bc86cd5853353", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bcftools/annotate": { "branch": "master", "git_sha": "2ad29c2aed06d815d9f68ad7ba20b3b1c574ce9c", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/bcftools/annotate/bcftools-annotate.diff" }, "bcftools/concat": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bcftools/filter": { "branch": "master", "git_sha": "a3893076a76e91b3ff152faddf872f00778fb224", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/bcftools/filter/bcftools-filter.diff" }, "bcftools/sort": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bedgovcf": { "branch": "master", "git_sha": "71a96063a7e09be7ea48f069d134a56ce86c9ba6", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bwa/index": { "branch": "master", "git_sha": "6278bf9afd4a4b2d00fa6052250e73da3d91546f", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "delly/call": { "branch": "master", "git_sha": "92bf2a8afa318a64f9d333cf87b510c662fd4274", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/delly/call/delly-call.diff" }, "ensemblvep/download": { "branch": "master", "git_sha": "3db4f8488315cd7d7cf3fcb64251f6603210e831", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "ensemblvep/vep": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "expansionhunter": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gawk": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gridss/gridss": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/gridss/gridss/gridss-gridss.diff" }, "jasminesv": { "branch": "master", "git_sha": "92bf2a8afa318a64f9d333cf87b510c662fd4274", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/jasminesv/jasminesv.diff" }, "manta/convertinversion": { "branch": "master", "git_sha": "92bf2a8afa318a64f9d333cf87b510c662fd4274", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "manta/germline": { "branch": "master", "git_sha": "ebc1733b77c702f19fe42076a5edfcbaa0d84f66", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "multiqc": { "branch": "master", - "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", - "installed_by": ["modules"] + "git_sha": "19ca321db5d8bd48923262c2eca6422359633491", + "installed_by": [ + "modules" + ] }, "ngsbits/samplegender": { "branch": "master", "git_sha": "023e119cfa102f319b6dcfd06abecf7f3c2b8df1", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "samtools/convert": { "branch": "master", "git_sha": "b42fec6f7c6e5d0716685cabb825ef6bf6e386b5", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/samtools/convert/samtools-convert.diff" }, "samtools/faidx": { "branch": "master", "git_sha": "f153f1f10e1083c49935565844cccb7453021682", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "samtools/index": { "branch": "master", "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "samtools/merge": { "branch": "master", "git_sha": "f4596fe0bdc096cf53ec4497e83defdb3a94ff62", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "smoove/call": { "branch": "master", "git_sha": "f2fdc5e81efe36beff213752ce1a8e71fceeacba", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "svync": { "branch": "master", "git_sha": "3df1a372b844c98c31ec3446faac7c75e64518a3", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "tabix/bgziptabix": { "branch": "master", "git_sha": "09d3c8c29b31a2dfd610305b10550f0e1dbcd4a9", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "tabix/tabix": { "branch": "master", "git_sha": "9502adb23c0b97ed8e616bbbdfa73b4585aec9a1", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "untar": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "vcfanno": { "branch": "master", "git_sha": "b558cd24f8751dcda51f957391f5f9cb83e28586", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "wisecondorx/convert": { "branch": "master", "git_sha": "dc7e9607d9977555ca1235a51a40d097bc66954c", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "wisecondorx/predict": { "branch": "master", "git_sha": "dc7e9607d9977555ca1235a51a40d097bc66954c", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] } } }, @@ -172,21 +234,27 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["subworkflows"] + "git_sha": "d20fb2a9cc3e2835e9d067d1046a63252eb17352", + "installed_by": [ + "subworkflows" + ] }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["subworkflows"] + "git_sha": "2fdce49d30c0254f76bc0f13c55c17455c1251ab", + "installed_by": [ + "subworkflows" + ] }, - "utils_nfvalidation_plugin": { + "utils_nfschema_plugin": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["subworkflows"] + "git_sha": "bbd5a41f4535a8defafe6080e00ea74c45f4f96c", + "installed_by": [ + "subworkflows" + ] } } } } } -} +} \ No newline at end of file diff --git a/modules/local/bcftools/consensus_reheader/main.nf b/modules/local/bcftools/consensus_reheader/main.nf index c1c885a6..636f58f5 100644 --- a/modules/local/bcftools/consensus_reheader/main.nf +++ b/modules/local/bcftools/consensus_reheader/main.nf @@ -16,9 +16,6 @@ process BCFTOOLS_CONSENSUS_REHEADER { tuple val(meta), path("*.${extension}"), emit: vcf path "versions.yml" , emit: versions - when: - task.ext.when == null || task.ext.when - script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/modules/local/bcftools/split_by_svtype/main.nf b/modules/local/bcftools/split_by_svtype/main.nf index 6af03d81..05f9d155 100644 --- a/modules/local/bcftools/split_by_svtype/main.nf +++ b/modules/local/bcftools/split_by_svtype/main.nf @@ -15,9 +15,6 @@ process BCFTOOLS_SPLIT_BY_SVTYPE { tuple val(meta), path("*.txt") , emit: header path "versions.yml" , emit: versions - when: - task.ext.when == null || task.ext.when - script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/modules/local/fix_callers/main.nf b/modules/local/fix_callers/main.nf index 578e4adb..4ce490a2 100644 --- a/modules/local/fix_callers/main.nf +++ b/modules/local/fix_callers/main.nf @@ -11,9 +11,6 @@ process FIX_CALLERS { tuple val(meta), path("*.vcf.gz") , emit: vcf path "versions.yml" , emit: versions - when: - task.ext.when == null || task.ext.when - script: def prefix = task.ext.prefix ?: "${meta.id}" diff --git a/modules/local/qdnaseq/main.nf b/modules/local/qdnaseq/main.nf index c747b93d..b38a7d8e 100644 --- a/modules/local/qdnaseq/main.nf +++ b/modules/local/qdnaseq/main.nf @@ -15,9 +15,6 @@ process QDNASEQ { tuple val(meta), path("statistics.out") , emit: statistics path "versions.yml" , emit: versions - when: - task.ext.when == null || task.ext.when - script: template "qDNAseq.R" diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index ca39fb67..a31464c9 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.21 + - bioconda::multiqc=1.24.1 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 47ac352f..ceaec139 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,14 +3,16 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : - 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.24.1--pyhdfd78af_0' : + 'biocontainers/multiqc:1.24.1--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" path(multiqc_config) path(extra_multiqc_config) path(multiqc_logo) + path(replace_names) + path(sample_names) output: path "*multiqc_report.html", emit: report @@ -23,16 +25,22 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ? "--filename ${task.ext.prefix}.html" : '' def config = multiqc_config ? "--config $multiqc_config" : '' def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' - def logo = multiqc_logo ? /--cl-config 'custom_logo: "${multiqc_logo}"'/ : '' + def logo = multiqc_logo ? "--cl-config 'custom_logo: \"${multiqc_logo}\"'" : '' + def replace = replace_names ? "--replace-names ${replace_names}" : '' + def samples = sample_names ? "--sample-names ${sample_names}" : '' """ multiqc \\ --force \\ $args \\ $config \\ + $prefix \\ $extra_config \\ $logo \\ + $replace \\ + $samples \\ . cat <<-END_VERSIONS > versions.yml diff --git a/modules/nf-core/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml index 45a9bc35..382c08cb 100644 --- a/modules/nf-core/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -29,6 +29,19 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" + - replace_names: + type: file + description: | + Optional two-column sample renaming file. First column a set of + patterns, second column a set of corresponding replacements. Passed via + MultiQC's `--replace-names` option. + pattern: "*.{tsv}" + - sample_names: + type: file + description: | + Optional TSV file with headers, passed to the MultiQC --sample_names + argument. + pattern: "*.{tsv}" output: - report: type: file diff --git a/modules/nf-core/multiqc/tests/main.nf.test b/modules/nf-core/multiqc/tests/main.nf.test index f1c4242e..33316a7d 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test +++ b/modules/nf-core/multiqc/tests/main.nf.test @@ -8,6 +8,8 @@ nextflow_process { tag "modules_nfcore" tag "multiqc" + config "./nextflow.config" + test("sarscov2 single-end [fastqc]") { when { @@ -17,6 +19,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -41,6 +45,8 @@ nextflow_process { input[1] = Channel.of(file("https://github.com/nf-core/tools/raw/dev/nf_core/pipeline-template/assets/multiqc_config.yml", checkIfExists: true)) input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } @@ -66,6 +72,8 @@ nextflow_process { input[1] = [] input[2] = [] input[3] = [] + input[4] = [] + input[5] = [] """ } } diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index bfebd802..83fa080c 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,6eb13f3b11bbcbfc98ad3166420ff760" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:48:55.657331" + "timestamp": "2024-07-10T12:41:34.562023" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,6eb13f3b11bbcbfc98ad3166420ff760" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:49.071937" + "timestamp": "2024-07-10T11:27:11.933869532" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" + "versions.yml:md5,6eb13f3b11bbcbfc98ad3166420ff760" ] ], "meta": { "nf-test": "0.8.4", - "nextflow": "23.10.1" + "nextflow": "24.04.2" }, - "timestamp": "2024-02-29T08:49:25.457567" + "timestamp": "2024-07-10T11:26:56.709849369" } -} \ No newline at end of file +} diff --git a/modules/nf-core/multiqc/tests/nextflow.config b/modules/nf-core/multiqc/tests/nextflow.config new file mode 100644 index 00000000..c537a6a3 --- /dev/null +++ b/modules/nf-core/multiqc/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: 'MULTIQC' { + ext.prefix = null + } +} diff --git a/nextflow.config b/nextflow.config index 182c8cf1..40c7882d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -63,6 +63,7 @@ params { cmgg_config_base = "/conf/" igenomes_base = null //'s3://ngi-igenomes/igenomes' igenomes_ignore = true + genomes = [:] // Boilerplate options outdir = null @@ -73,7 +74,6 @@ params { hook_url = null help = false version = false - monochromeLogs = false // Config options config_profile_name = null @@ -83,13 +83,8 @@ params { config_profile_contact = null config_profile_url = null - // Max resource options - // Defaults only, expecting to be overwritten - max_memory = '128.GB' - max_cpus = 16 - max_time = '240.h' - // Schema validation default options + monochrome_logs = false validationFailUnrecognisedParams = false validationLenientMode = false validationSchemaIgnoreParams = 'genomes,igenomes_base,test_data' @@ -102,22 +97,7 @@ params { includeConfig 'conf/base.config' // Load nf-core custom profiles from different Institutions -try { - includeConfig "${params.custom_config_base}/nfcore_custom.config" -} catch (Exception e) { - System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") -} - -// Load nf-cmgg/structural custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/structural.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/structural profiles: ${params.custom_config_base}/pipeline/structural.config") -// } - -// Include test data config -includeConfig 'https://raw.githubusercontent.com/nf-cmgg/test-datasets/main/nf-cmgg-testdata.config' +includeConfig !System.getenv('NXF_OFFLINE') ? "${params.custom_config_base}/nfcore_custom.config" : "conf/empty.config" profiles { debug { @@ -133,7 +113,6 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { @@ -224,19 +203,9 @@ docker.registry = 'quay.io' podman.registry = 'quay.io' singularity.registry = 'quay.io' -// Nextflow plugins -plugins { - id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet -} -// Load igenomes.config if required -if (!params.igenomes_ignore) { - includeConfig "conf/igenomes.config" -} else if (!params.genomes_ignore){ - includeConfig "https://raw.githubusercontent.com/nf-cmgg/configs/main/conf/Hsapiens/genomes.config" -} else { - params.genomes = [:] -} +includeConfig !params.igenomes_ignore ? "conf/igenomes.config" : !params.genomes_ignore ? "https://raw.githubusercontent.com/nf-cmgg/configs/main/conf/Hsapiens/genomes.config" : "conf/empty_genomes.config" + // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. @@ -254,22 +223,21 @@ process.shell = ['/bin/bash', '-euo', 'pipefail'] // Disable process selector warnings by default. Use debug profile to enable warnings. nextflow.enable.configProcessNamesValidation = false -def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true - file = "${params.outdir}/pipeline_info/execution_timeline_${trace_timestamp}.html" + file = "${params.outdir}/pipeline_info/execution_timeline_${new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')}.html" } report { enabled = true - file = "${params.outdir}/pipeline_info/execution_report_${trace_timestamp}.html" + file = timeline.file.replace("timeline", "report") } trace { enabled = true - file = "${params.outdir}/pipeline_info/execution_trace_${trace_timestamp}.txt" + file = timeline.file.replace("timeline", "trace") } dag { enabled = true - file = "${params.outdir}/pipeline_info/pipeline_dag_${trace_timestamp}.html" + file = timeline.file.replace("execution_timeline", "pipeline_dag") } manifest { @@ -278,43 +246,36 @@ manifest { homePage = 'https://github.com/nf-cmgg/structural' description = 'A bioinformatics best-practice analysis pipeline for calling structural variants (SVs), copy number variants (CNVs) and repeat region expansions (RREs) from short DNA reads' mainScript = 'main.nf' - nextflowVersion = '!>=23.10.0' + nextflowVersion = '!>=24.04.0' version = '0.3.0dev' doi = '' } -// Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' +// Nextflow plugins +plugins { + id 'nf-schema@2.1.0' // Validation of pipeline parameters and creation of an input channel from a sample sheet +} -// Function to ensure that resource requirements don't go beyond -// a maximum limit -def check_max(obj, type) { - if (type == 'memory') { - try { - if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1) - return params.max_memory as nextflow.util.MemoryUnit - else - return obj - } catch (all) { - println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'time') { - try { - if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1) - return params.max_time as nextflow.util.Duration - else - return obj - } catch (all) { - println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj" - return obj - } - } else if (type == 'cpus') { - try { - return Math.min( obj, params.max_cpus as int ) - } catch (all) { - println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj" - return obj - } +validation { + defaultIgnoreParams = ['genomes','igenomes_base','test_data'] + help { + enabled = true + beforeText = """ +-\033[2m----------------------------------------------------\033[0m- + \033[0;34m ///\033[0;32m/// \033[0m +\033[0;34m ___ __ _ _ __ __ \033[0;34m ///\033[0;32m///// \033[0m +\033[0;34m |\\ | |__ __ / ` | \\/ | / _` / _` \033[0;34m////\033[0;32m////// \033[0m +\033[0;34m | \\| | \\__, | | \\__| \\__| \033[0;34m///\033[0;32m///// \033[0m + \033[0;34m///\033[0;32m/// \033[0m +\033[0;35m ${manifest.name} ${manifest.version}\033[0m +-\033[2m----------------------------------------------------\033[0m- +""" + command = "nextflow run ${manifest.name} -profile docker --input --outdir " + } + summary { + beforeText = validation.help.beforeText } } + +// Load modules.config for DSL2 module specific options +includeConfig 'conf/modules.config' diff --git a/nextflow_schema.json b/nextflow_schema.json index ecb3490e..cb82522a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/nf-cmgg/structural/master/nextflow_schema.json", "title": "nf-cmgg/structural pipeline parameters", - "description": "A nextflow pipeline for calling structural variants", + "description": "A bioinformatics best-practice analysis pipeline for calling structural variants (SVs), copy number variants (CNVs) and repeat region expansions (RREs) from short DNA reads", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -135,6 +135,11 @@ "default": "/conf/", "description": "The config base path for the cmgg configs", "format": "directory-path" + }, + "genomes": { + "type": "object", + "description": "A map containing all references for all genomes", + "hidden": true } } }, @@ -186,41 +191,6 @@ } } }, - "max_job_request_options": { - "title": "Max job request options", - "type": "object", - "fa_icon": "fab fa-acquisitions-incorporated", - "description": "Set the top limit for requested resources for any single job.", - "help_text": "If you are running on a smaller system, a pipeline step requesting more resources than are available may cause the Nextflow to stop the run with an error. These options allow you to cap the maximum resources requested by any single job so that the pipeline will run on your system.\n\nNote that you can not _increase_ the resources requested by any job using these options. For that you will need your own configuration file. See [the nf-core website](https://nf-co.re/usage/configuration) for details.", - "properties": { - "max_cpus": { - "type": "integer", - "description": "Maximum number of CPUs that can be requested for any single job.", - "default": 16, - "fa_icon": "fas fa-microchip", - "hidden": true, - "help_text": "Use to set an upper-limit for the CPU requirement for each process. Should be an integer e.g. `--max_cpus 1`" - }, - "max_memory": { - "type": "string", - "description": "Maximum amount of memory that can be requested for any single job.", - "default": "128.GB", - "fa_icon": "fas fa-memory", - "pattern": "^\\d+(\\.\\d+)?\\.?\\s*(K|M|G|T)?B$", - "hidden": true, - "help_text": "Use to set an upper-limit for the memory requirement for each process. Should be a string in the format integer-unit e.g. `--max_memory '8.GB'`" - }, - "max_time": { - "type": "string", - "description": "Maximum amount of time that can be requested for any single job.", - "default": "240.h", - "fa_icon": "far fa-clock", - "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$", - "hidden": true, - "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`" - } - } - }, "generic_options": { "title": "Generic options", "type": "object", @@ -304,6 +274,10 @@ "default": true, "fa_icon": "fas fa-check-square", "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Output monochrome logs" } } }, @@ -501,40 +475,37 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" - }, - { - "$ref": "#/definitions/reference_genome_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/institutional_config_options" + "$ref": "#/$defs/reference_genome_options" }, { - "$ref": "#/definitions/max_job_request_options" + "$ref": "#/$defs/institutional_config_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" }, { - "$ref": "#/definitions/pipeline_specific_options" + "$ref": "#/$defs/pipeline_specific_options" }, { - "$ref": "#/definitions/delly_parameters" + "$ref": "#/$defs/delly_parameters" }, { - "$ref": "#/definitions/manta_parameters" + "$ref": "#/$defs/manta_parameters" }, { - "$ref": "#/definitions/qdnaseq_parameters" + "$ref": "#/$defs/qdnaseq_parameters" }, { - "$ref": "#/definitions/vep_options" + "$ref": "#/$defs/vep_options" }, { - "$ref": "#/definitions/annotsv_options" + "$ref": "#/$defs/annotsv_options" }, { - "$ref": "#/definitions/vcfanno_options" + "$ref": "#/$defs/vcfanno_options" } ] } diff --git a/subworkflows/local/bam_cnv_calling/main.nf b/subworkflows/local/bam_cnv_calling/main.nf index b4f8fb2c..5179d84e 100644 --- a/subworkflows/local/bam_cnv_calling/main.nf +++ b/subworkflows/local/bam_cnv_calling/main.nf @@ -54,9 +54,10 @@ workflow BAM_CNV_CALLING { ch_called_vcfs = ch_called_vcfs.mix(BAM_VARIANT_CALLING_WISECONDORX.out.vcf) } + ch_merged_vcfs = Channel.empty() if(val_callers.size() > 1) { VCF_MERGE_CALLERS_JASMINE( - ch_called_vcfs.map { it + [[]] }, + ch_called_vcfs.map { meta, vcf -> [meta, vcf, []] }, ch_fasta, ch_fai, val_callers, diff --git a/subworkflows/local/bam_sv_calling/main.nf b/subworkflows/local/bam_sv_calling/main.nf index 471f4b29..8970a01f 100644 --- a/subworkflows/local/bam_sv_calling/main.nf +++ b/subworkflows/local/bam_sv_calling/main.nf @@ -97,6 +97,7 @@ workflow BAM_SV_CALLING { // Scramble is unfinished. It needs a lot of improvements if we were to add it + ch_merged_vcfs = Channel.empty() if(val_callers.size() > 1) { VCF_MERGE_CALLERS_JASMINE( ch_called_vcfs, @@ -116,7 +117,6 @@ workflow BAM_SV_CALLING { .set { ch_merged_vcfs } } - emit: vcfs = ch_merged_vcfs // channel: [ val(meta), path(vcf), path(tbi) ] diff --git a/subworkflows/local/bam_variant_calling_delly/main.nf b/subworkflows/local/bam_variant_calling_delly/main.nf index e7045483..2100812c 100644 --- a/subworkflows/local/bam_variant_calling_delly/main.nf +++ b/subworkflows/local/bam_variant_calling_delly/main.nf @@ -36,8 +36,8 @@ workflow BAM_VARIANT_CALLING_DELLY { ch_versions = ch_versions.mix(DELLY_CALL.out.versions.first()) ch_svync_configs - .map { - it.find { it.toString().contains("delly") } + .map { configs -> + configs.find { config -> config.toString().contains("delly") } } .set { ch_delly_svync_config } diff --git a/subworkflows/local/bam_variant_calling_gridss/main.nf b/subworkflows/local/bam_variant_calling_gridss/main.nf index 206d0f14..f330720a 100644 --- a/subworkflows/local/bam_variant_calling_gridss/main.nf +++ b/subworkflows/local/bam_variant_calling_gridss/main.nf @@ -1,58 +1,58 @@ -// -// Run Gridss -// - -include { GRIDSS_GRIDSS } from '../../../modules/nf-core/gridss/gridss/main' -include { TABIX_TABIX } from '../../../modules/nf-core/tabix/tabix/main' -include { VIOLA } from '../../../modules/local/viola/main' -include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort/main' - - -workflow BAM_VARIANT_CALLING_GRIDSS { - take: - ch_crams // channel: [mandatory] [ meta, cram, crai ] => The aligned CRAMs per sample with the regions they should be called on - ch_fasta // channel: [mandatory] [ meta, fasta ] => The fasta reference file - ch_fai // channel: [mandatory] [ meta, fai ] => The index of the fasta reference file - ch_bwa_index // channel: [mandatory] [ meta, index ] => The BWA MEM index - - main: - - ch_versions = Channel.empty() - - GRIDSS_GRIDSS( - ch_crams.map {meta, cram, crai -> [meta, cram, []]}, - ch_fasta, - ch_fai, - ch_bwa_index - ) - ch_versions = ch_versions.mix(GRIDSS_GRIDSS.out.versions.first()) - - VIOLA( - GRIDSS_GRIDSS.out.vcf, - "gridss" - ) - ch_versions = ch_versions.mix(VIOLA.out.versions.first()) - - BCFTOOLS_SORT( - VIOLA.out.vcf - ) - ch_versions = ch_versions.mix(BCFTOOLS_SORT.out.versions.first()) - - TABIX_TABIX( - BCFTOOLS_SORT.out.vcf - ) - ch_versions = ch_versions.mix(TABIX_TABIX.out.versions.first()) - - BCFTOOLS_SORT.out.vcf - .join(TABIX_TABIX.out.tbi, failOnMismatch:true, failOnDuplicate:true) - .map{ meta, vcf, tbi -> - new_meta = meta + [caller:"gridss"] - [ new_meta, vcf, tbi ] - } - .dump(tag: 'gridss_vcfs', pretty: true) - .set { ch_gridss_vcfs } - - emit: - gridss_vcfs = ch_gridss_vcfs // channel: [ val(meta), path(vcf), path(tbi) ] - versions = ch_versions -} +// // +// // Run Gridss +// // + +// include { GRIDSS_GRIDSS } from '../../../modules/nf-core/gridss/gridss/main' +// include { TABIX_TABIX } from '../../../modules/nf-core/tabix/tabix/main' +// include { VIOLA } from '../../../modules/local/viola/main' +// include { BCFTOOLS_SORT } from '../../../modules/nf-core/bcftools/sort/main' + + +// workflow BAM_VARIANT_CALLING_GRIDSS { +// take: +// ch_crams // channel: [mandatory] [ meta, cram, crai ] => The aligned CRAMs per sample with the regions they should be called on +// ch_fasta // channel: [mandatory] [ meta, fasta ] => The fasta reference file +// ch_fai // channel: [mandatory] [ meta, fai ] => The index of the fasta reference file +// ch_bwa_index // channel: [mandatory] [ meta, index ] => The BWA MEM index + +// main: + +// ch_versions = Channel.empty() + +// GRIDSS_GRIDSS( +// ch_crams.map {meta, cram, crai -> [meta, cram, []]}, +// ch_fasta, +// ch_fai, +// ch_bwa_index +// ) +// ch_versions = ch_versions.mix(GRIDSS_GRIDSS.out.versions.first()) + +// VIOLA( +// GRIDSS_GRIDSS.out.vcf, +// "gridss" +// ) +// ch_versions = ch_versions.mix(VIOLA.out.versions.first()) + +// BCFTOOLS_SORT( +// VIOLA.out.vcf +// ) +// ch_versions = ch_versions.mix(BCFTOOLS_SORT.out.versions.first()) + +// TABIX_TABIX( +// BCFTOOLS_SORT.out.vcf +// ) +// ch_versions = ch_versions.mix(TABIX_TABIX.out.versions.first()) + +// BCFTOOLS_SORT.out.vcf +// .join(TABIX_TABIX.out.tbi, failOnMismatch:true, failOnDuplicate:true) +// .map{ meta, vcf, tbi -> +// new_meta = meta + [caller:"gridss"] +// [ new_meta, vcf, tbi ] +// } +// .dump(tag: 'gridss_vcfs', pretty: true) +// .set { ch_gridss_vcfs } + +// emit: +// gridss_vcfs = ch_gridss_vcfs // channel: [ val(meta), path(vcf), path(tbi) ] +// versions = ch_versions +// } diff --git a/subworkflows/local/bam_variant_calling_manta/main.nf b/subworkflows/local/bam_variant_calling_manta/main.nf index ff4e71e0..95ac9b20 100644 --- a/subworkflows/local/bam_variant_calling_manta/main.nf +++ b/subworkflows/local/bam_variant_calling_manta/main.nf @@ -72,8 +72,8 @@ workflow BAM_VARIANT_CALLING_MANTA { ch_versions = ch_versions.mix(MANTA_CONVERTINVERSION.out.versions.first()) ch_svync_configs - .map { - it.find { it.toString().contains("manta") } + .map { configs -> + configs.find { config -> config.toString().contains("manta") } } .set { ch_manta_svync_config } diff --git a/subworkflows/local/bam_variant_calling_qdnaseq/main.nf b/subworkflows/local/bam_variant_calling_qdnaseq/main.nf index 1643d7a3..702a4f98 100644 --- a/subworkflows/local/bam_variant_calling_qdnaseq/main.nf +++ b/subworkflows/local/bam_variant_calling_qdnaseq/main.nf @@ -67,8 +67,8 @@ workflow BAM_VARIANT_CALLING_QDNASEQ { ch_versions = ch_versions.mix(GAWK.out.versions.first()) ch_bedgovcf_configs - .map { - it.find { it.toString().contains("qdnaseq") } + .map { configs -> + configs.find { config -> config.toString().contains("qdnaseq") } } .set { ch_qdnaseq_bedgovcf_config } diff --git a/subworkflows/local/bam_variant_calling_smoove/main.nf b/subworkflows/local/bam_variant_calling_smoove/main.nf index 9dede86e..2a855411 100644 --- a/subworkflows/local/bam_variant_calling_smoove/main.nf +++ b/subworkflows/local/bam_variant_calling_smoove/main.nf @@ -49,8 +49,8 @@ workflow BAM_VARIANT_CALLING_SMOOVE { ch_versions = ch_versions.mix(TABIX_CALLER.out.versions.first()) ch_svync_configs - .map { - it.find { it.toString().contains("smoove") } + .map { configs -> + configs.find { config -> config.toString().contains("smoove") } } .set { ch_smoove_svync_config } diff --git a/subworkflows/local/bam_variant_calling_wisecondorx/main.nf b/subworkflows/local/bam_variant_calling_wisecondorx/main.nf index 61ef91ea..f283ab53 100644 --- a/subworkflows/local/bam_variant_calling_wisecondorx/main.nf +++ b/subworkflows/local/bam_variant_calling_wisecondorx/main.nf @@ -39,8 +39,8 @@ workflow BAM_VARIANT_CALLING_WISECONDORX { ch_versions = ch_versions.mix(WISECONDORX_PREDICT.out.versions.first()) ch_bedgovcf_configs - .map { - it.find { it.toString().contains("wisecondorx") } + .map { configs -> + configs.find { config -> config.toString().contains("wisecondorx") } } .set { ch_wisecondorx_bedgovcf_config } diff --git a/subworkflows/local/utils_nfcore_structural_pipeline/main.nf b/subworkflows/local/utils_nfcore_structural_pipeline/main.nf index 7bd9d727..9b81cfcb 100644 --- a/subworkflows/local/utils_nfcore_structural_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_structural_pipeline/main.nf @@ -8,9 +8,9 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' -include { paramsSummaryMap } from 'plugin/nf-validation' -include { fromSamplesheet } from 'plugin/nf-validation' +include { UTILS_NFSCHEMA_PLUGIN } from '../../nf-core/utils_nfschema_plugin' +include { paramsSummaryMap } from 'plugin/nf-schema' +include { samplesheetToList } from 'plugin/nf-schema' include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' @@ -36,6 +36,8 @@ workflow PIPELINE_INITIALISATION { nextflow_cli_args // array: List of positional nextflow CLI args outdir // string: The output directory where the results will be saved input // string: Path to input samplesheet + genome // string: The genome to be used + genomes // map: A map containing all references for each genome main: @@ -56,14 +58,11 @@ workflow PIPELINE_INITIALISATION { // pre_help_text = nfCoreLogo(monochrome_logs) post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " - UTILS_NFVALIDATION_PLUGIN ( - help, - workflow_command, - pre_help_text, - post_help_text, + def workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " + UTILS_NFSCHEMA_PLUGIN ( + workflow, validate_params, - "nextflow_schema.json" + [] ) // @@ -75,16 +74,16 @@ workflow PIPELINE_INITIALISATION { // // Custom validation for pipeline parameters // - validateInputParameters() + validateInputParameters(genome, genomes) // // Create channel from input file provided through params.input // - Channel.fromSamplesheet("input") - .map { - def meta = it[0] + Channel.fromList(samplesheetToList(input, "assets/schema_input.json")) + .map { row -> + def meta = row[0] def new_meta = meta.family ? meta : meta + [family:meta.sample] - return [ new_meta.family, new_meta ] + it.subList(1, it.size()) + return [ new_meta.family, new_meta ] + row.subList(1, row.size()) } .tap { ch_raw_input } .reduce([:]) { counts, entry -> @@ -94,9 +93,9 @@ workflow PIPELINE_INITIALISATION { return counts } .combine(ch_raw_input) - .map { // counts, family, meta, ... - it[2] = it[2] + ["family_count":it[0][it[1]].size()] - return it.subList(2, it.size()) + .map { row -> // counts, family, meta, ... + row[2] = row[2] + ["family_count":row[0][row[1]].size()] + return row.subList(2, row.size()) } .set { ch_samplesheet } @@ -150,31 +149,17 @@ workflow PIPELINE_COMPLETION { // // Check and validate pipeline parameters // -def validateInputParameters() { - genomeExistsError() +def validateInputParameters(String genome, Map genomes) { + genomeExistsError(genome, genomes) } -// -// Validate channels from input samplesheet -// -def validateInputSamplesheet(input) { - def (metas, fastqs) = input[1..2] - - // Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end - def endedness_ok = metas.collect{ it.single_end }.unique().size == 1 - if (!endedness_ok) { - error("Please check input samplesheet -> Multiple runs of a sample must be of the same datatype i.e. single-end or paired-end: ${metas[0].id}") - } - - return [ metas[0], fastqs ] -} // // Get attribute from genome config file e.g. fasta // -def getGenomeAttribute(attribute) { - if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { - if (params.genomes[ params.genome ].containsKey(attribute)) { - return params.genomes[ params.genome ][ attribute ] +def getGenomeAttribute(String attribute, String genome, Map genomes) { + if (genomes && genome && genomes.containsKey(genome)) { + if (genomes[ genome ].containsKey(attribute)) { + return genomes[ genome ][ attribute ] } } return null @@ -183,12 +168,12 @@ def getGenomeAttribute(attribute) { // // Exit pipeline if incorrect --genome key provided // -def genomeExistsError() { - if (params.genomes && params.genome && !params.genomes.containsKey(params.genome)) { +def genomeExistsError(String genome, Map genomes) { + if (genomes && genome && !genomes.containsKey(genome)) { def error_string = "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " Genome '${params.genome}' not found in any config files provided to the pipeline.\n" + + " Genome '${genome}' not found in any config files provided to the pipeline.\n" + " Currently, the available genome keys are:\n" + - " ${params.genomes.keySet().join(", ")}\n" + + " ${genomes.keySet().join(", ")}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" error(error_string) } diff --git a/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf b/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf index 320f4557..e94df7ad 100644 --- a/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf +++ b/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf @@ -205,10 +205,10 @@ workflow VCF_ANNOTATE_VEP_ANNOTSV_VCFANNO { def create_vcfanno_toml(vcfanno_resources, input_vcfanno_toml, List vcfanno_defaults) { def vcfanno_toml = input_vcfanno_toml ? parse_toml(input_vcfanno_toml) : [:] def default_tomls = parse_toml(vcfanno_defaults) - def resources = vcfanno_resources.collect { it.fileName.toString() } + def resources = vcfanno_resources.collect { Path resource -> resource.fileName.toString() } resources.add("annotsv_annotated.vcf.gz" as String) def output = [] - for (file_name in resources) { + resources.each { file_name -> if (vcfanno_toml.containsKey(file_name)){ output.add(vcfanno_toml[file_name]) } @@ -216,7 +216,7 @@ def create_vcfanno_toml(vcfanno_resources, input_vcfanno_toml, List vcfann output.add(default_tomls[file_name]) } } - postannotation = vcfanno_toml.postannotation != [] ? vcfanno_toml.postannotation : default_tomls.postannotation + def postannotation = vcfanno_toml.postannotation != [] ? vcfanno_toml.postannotation : default_tomls.postannotation if (postannotation != []){ output.add(postannotation) } @@ -226,13 +226,13 @@ def create_vcfanno_toml(vcfanno_resources, input_vcfanno_toml, List vcfann def parse_toml(tomls) { def output = [:] output.postannotation = [] - toml_list = tomls instanceof List ? tomls : [tomls] - for (toml in toml_list) { + def toml_list = tomls instanceof List ? tomls : [tomls] + toml_list.each { toml -> def info = "" def file = "" def fields = [] - for (line in toml.readLines()) { - if (line.startsWith("#")) { continue } + toml.readLines().each { line -> + if (line.startsWith("#")) { return } if (line == "[[annotation]]" || line == "[[postannotation]]") { if (info.startsWith("[[postannotation]]")) { output.postannotation.add(create_toml_config(fields)) @@ -262,6 +262,6 @@ def parse_toml(tomls) { } def create_toml_config(fields_list) { - config = fields_list.findAll { it != "" }.join("\n") + def config = fields_list.findAll { field -> field != "" }.join("\n") return "${config}\n" } diff --git a/subworkflows/local/vcf_merge_family_jasmine/main.nf b/subworkflows/local/vcf_merge_family_jasmine/main.nf index e429c00c..37b0f5a8 100644 --- a/subworkflows/local/vcf_merge_family_jasmine/main.nf +++ b/subworkflows/local/vcf_merge_family_jasmine/main.nf @@ -20,7 +20,7 @@ workflow VCF_MERGE_FAMILY_JASMINE { ch_versions = Channel.empty() ch_vcfs - .filter { it[0].family_count > 1 } + .filter { meta, vcf, tbi -> meta.family_count > 1 } .map { meta, vcf, tbi -> def new_meta = meta - meta.subMap("sample", "sex") + ["id":meta.variant_type ? "${meta.family}.${meta.variant_type}" : meta.family] [ groupKey(new_meta, meta.family_count), vcf, tbi ] @@ -32,12 +32,12 @@ workflow VCF_MERGE_FAMILY_JASMINE { } .tap { ch_meta_file_list } .map { id, meta, vcfs -> - [ "${id}_list.txt", vcfs.collect { it.baseName }.join("\n") ] + [ "${id}_list.txt", vcfs.collect { vcf -> vcf.baseName }.join("\n") ] } .collectFile() - .map { - def id = it.name.replaceAll("_list.txt\$", "") - [ id, it ] + .map { meta_file -> + def id = meta_file.name.replaceAll("_list.txt\$", "") + [ id, meta_file ] } .join(ch_meta_file_list, failOnMismatch:true, failOnDuplicate:true) .map { id, file_list, meta, vcfs -> diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf index ac31f28f..28e32b20 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -2,10 +2,6 @@ // Subworkflow with functionality that may be useful for any Nextflow pipeline // -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -58,7 +54,7 @@ workflow UTILS_NEXTFLOW_PIPELINE { // Generate version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -79,10 +75,10 @@ def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') def filename = "params_${timestamp}.json" def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) + def jsonStr = groovy.json.JsonOutput.toJson(params) + temp_pf.text = groovy.json.JsonOutput.prettyPrint(jsonStr) - FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + nextflow.extension.FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") temp_pf.delete() } @@ -90,7 +86,7 @@ def dumpParametersToJSON(outdir) { // When running with -profile conda, warn if channels have not been set-up appropriately // def checkCondaChannels() { - Yaml parser = new Yaml() + def parser = new org.yaml.snakeyaml.Yaml() def channels = [] try { def config = parser.load("conda config --show channels".execute().text) @@ -102,14 +98,16 @@ def checkCondaChannels() { // Check that all channels are present // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def required_channels_in_order = ['conda-forge', 'bioconda'] def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + + required_channels_in_order.eachWithIndex { channel, index -> + if (index < required_channels_in_order.size() - 1) { + channel_priority_violation |= !(channels.indexOf(channel) < channels.indexOf(required_channels_in_order[index+1])) + } } if (channels_missing | channel_priority_violation) { diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index d0a926bf..a09572e5 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -3,7 +3,7 @@ manifest { author = """nf-core""" homePage = 'https://127.0.0.1' description = """Dummy pipeline""" - nextflowVersion = '!>=23.04.0' + nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6f..cbd8495b 100644 --- a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -2,9 +2,6 @@ // Subworkflow with utility functions specific to the nf-core pipeline template // -import org.yaml.snakeyaml.Yaml -import nextflow.extension.FilesEx - /* ======================================================================================== SUBWORKFLOW DEFINITION @@ -34,7 +31,7 @@ workflow UTILS_NFCORE_PIPELINE { // Warn if a -profile or Nextflow config has not been provided to run the pipeline // def checkConfigProvided() { - valid_config = true + def valid_config = true as Boolean if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + @@ -65,9 +62,17 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + def manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + manifest_doi.each { doi_ref -> + temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" + } return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + @@ -78,7 +83,7 @@ def workflowCitation() { // Generate workflow version string // def getWorkflowVersion() { - String version_string = "" + def version_string = "" as String if (workflow.manifest.version) { def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' version_string += "${prefix_v}${workflow.manifest.version}" @@ -96,8 +101,8 @@ def getWorkflowVersion() { // Get software versions for pipeline // def processVersionsFromYAML(yaml_file) { - Yaml yaml = new Yaml() - versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + def yaml = new org.yaml.snakeyaml.Yaml() + def versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } return yaml.dumpAsMap(versions).trim() } @@ -118,7 +123,7 @@ def workflowVersionToYAML() { def softwareVersionsToYAML(ch_versions) { return ch_versions .unique() - .map { processVersionsFromYAML(it) } + .map { version -> processVersionsFromYAML(version) } .unique() .mix(Channel.of(workflowVersionToYAML())) } @@ -128,19 +133,19 @@ def softwareVersionsToYAML(ch_versions) { // def paramsSummaryMultiqc(summary_params) { def summary_section = '' - for (group in summary_params.keySet()) { + summary_params.keySet().each { group -> def group_params = summary_params.get(group) // This gets the parameters of that particular group if (group_params) { summary_section += "

$group

\n" summary_section += "
\n" - for (param in group_params.keySet()) { + group_params.keySet().sort().each { param -> summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" } summary_section += "
\n" } } - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + def yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" as String yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" @@ -155,7 +160,7 @@ def paramsSummaryMultiqc(summary_params) { // nf-core logo // def nfCoreLogo(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map String.format( """\n ${dashedLine(monochrome_logs)} @@ -174,7 +179,7 @@ def nfCoreLogo(monochrome_logs=true) { // Return dashed line // def dashedLine(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map return "-${colors.dim}----------------------------------------------------${colors.reset}-" } @@ -182,7 +187,7 @@ def dashedLine(monochrome_logs=true) { // ANSII colours used for terminal logging // def logColours(monochrome_logs=true) { - Map colorcodes = [:] + def colorcodes = [:] as Map // Reset / Meta colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" @@ -281,7 +286,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi } def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } @@ -338,10 +343,10 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi def sendmail_html = sendmail_template.toString() // Send the HTML e-mail - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (email_address) { try { - if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { throw new org.codehaus.groovy.GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") sendmail_tf.withWriter { w -> w << sendmail_html } @@ -358,13 +363,13 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Write summary e-mail HTML to a file def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + nextflow.extension.FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); output_hf.delete() // Write summary e-mail TXT to a file def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + nextflow.extension.FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); output_tf.delete() } @@ -372,7 +377,7 @@ def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdi // Print pipeline summary on completion // def completionSummary(monochrome_logs=true) { - Map colors = logColours(monochrome_logs) + def colors = logColours(monochrome_logs) as Map if (workflow.success) { if (workflow.stats.ignoredCount == 0) { log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" @@ -389,7 +394,7 @@ def completionSummary(monochrome_logs=true) { // def imNotification(summary_params, hook_url) { def summary = [:] - for (group in summary_params.keySet()) { + summary_params.keySet().sort().each { group -> summary << summary_params[group] } diff --git a/subworkflows/nf-core/utils_nfschema_plugin/main.nf b/subworkflows/nf-core/utils_nfschema_plugin/main.nf new file mode 100644 index 00000000..4994303e --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/main.nf @@ -0,0 +1,46 @@ +// +// Subworkflow that uses the nf-schema plugin to validate parameters and render the parameter summary +// + +include { paramsSummaryLog } from 'plugin/nf-schema' +include { validateParameters } from 'plugin/nf-schema' + +workflow UTILS_NFSCHEMA_PLUGIN { + + take: + input_workflow // workflow: the workflow object used by nf-schema to get metadata from the workflow + validate_params // boolean: validate the parameters + parameters_schema // string: path to the parameters JSON schema. + // this has to be the same as the schema given to `validation.parametersSchema` + // when this input is empty it will automatically use the configured schema or + // "${projectDir}/nextflow_schema.json" as default. This input should not be empty + // for meta pipelines + + main: + + // + // Print parameter summary to stdout. This will display the parameters + // that differ from the default given in the JSON schema + // + if(parameters_schema) { + log.info paramsSummaryLog(input_workflow, parameters_schema:parameters_schema) + } else { + log.info paramsSummaryLog(input_workflow) + } + + // + // Validate the parameters using nextflow_schema.json or the schema + // given via the validation.parametersSchema configuration option + // + if(validate_params) { + if(parameters_schema) { + validateParameters(parameters_schema:parameters_schema) + } else { + validateParameters() + } + } + + emit: + dummy_emit = true +} + diff --git a/subworkflows/nf-core/utils_nfschema_plugin/meta.yml b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml new file mode 100644 index 00000000..f7d9f028 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/meta.yml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "utils_nfschema_plugin" +description: Run nf-schema to validate parameters and create a summary of changed parameters +keywords: + - validation + - JSON schema + - plugin + - parameters + - summary +components: [] +input: + - input_workflow: + type: object + description: | + The workflow object of the used pipeline. + This object contains meta data used to create the params summary log + - validate_params: + type: boolean + description: Validate the parameters and error if invalid. + - parameters_schema: + type: string + description: | + Path to the parameters JSON schema. + This has to be the same as the schema given to the `validation.parametersSchema` config + option. When this input is empty it will automatically use the configured schema or + "${projectDir}/nextflow_schema.json" as default. The schema should not be given in this way + for meta pipelines. +output: + - dummy_emit: + type: boolean + description: Dummy emit to make nf-core subworkflows lint happy +authors: + - "@nvnieuwk" +maintainers: + - "@nvnieuwk" diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test new file mode 100644 index 00000000..842dc432 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -0,0 +1,117 @@ +nextflow_workflow { + + name "Test Subworkflow UTILS_NFSCHEMA_PLUGIN" + script "../main.nf" + workflow "UTILS_NFSCHEMA_PLUGIN" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/utils_nfschema_plugin" + tag "plugin/nf-schema" + + config "./nextflow.config" + + test("Should run nothing") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } + + test("Should run nothing - custom schema") { + + when { + + params { + test_data = '' + } + + workflow { + """ + validate_params = false + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should validate params - custom schema") { + + when { + + params { + test_data = '' + outdir = 1 + } + + workflow { + """ + validate_params = true + input[0] = workflow + input[1] = validate_params + input[2] = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config new file mode 100644 index 00000000..0907ac58 --- /dev/null +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow.config @@ -0,0 +1,8 @@ +plugins { + id "nf-schema@2.1.0" +} + +validation { + parametersSchema = "${projectDir}/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json" + monochromeLogs = true +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json similarity index 95% rename from subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json rename to subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json index 7626c1c9..331e0d2f 100644 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/nextflow_schema.json @@ -1,10 +1,10 @@ { - "$schema": "http://json-schema.org/draft-07/schema", + "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", "title": ". pipeline parameters", "description": "", "type": "object", - "definitions": { + "$defs": { "input_output_options": { "title": "Input/output options", "type": "object", @@ -87,10 +87,10 @@ }, "allOf": [ { - "$ref": "#/definitions/input_output_options" + "$ref": "#/$defs/input_output_options" }, { - "$ref": "#/definitions/generic_options" + "$ref": "#/$defs/generic_options" } ] } diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf deleted file mode 100644 index 2585b65d..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf +++ /dev/null @@ -1,62 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsHelp } from 'plugin/nf-validation' -include { paramsSummaryLog } from 'plugin/nf-validation' -include { validateParameters } from 'plugin/nf-validation' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow UTILS_NFVALIDATION_PLUGIN { - - take: - print_help // boolean: print help - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log - validate_params // boolean: validate parameters - schema_filename // path: JSON schema file, null to use default value - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters(parameters_schema: schema_filename) - } - - emit: - dummy_emit = true -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml deleted file mode 100644 index 3d4a6b04..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml +++ /dev/null @@ -1,44 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "UTILS_NFVALIDATION_PLUGIN" -description: Use nf-validation to initiate and validate a pipeline -keywords: - - utility - - pipeline - - initialise - - validation -components: [] -input: - - print_help: - type: boolean - description: | - Print help message and exit - - workflow_command: - type: string - description: | - The command to run the workflow e.g. "nextflow run main.nf" - - pre_help_text: - type: string - description: | - Text to print before the help message - - post_help_text: - type: string - description: | - Text to print after the help message - - validate_params: - type: boolean - description: | - Validate the parameters and error if invalid. - - schema_filename: - type: string - description: | - The filename of the schema to validate against. -output: - - dummy_emit: - type: boolean - description: | - Dummy emit to make nf-core subworkflows lint happy -authors: - - "@adamrtalbot" -maintainers: - - "@adamrtalbot" - - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test deleted file mode 100644 index 5784a33f..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test +++ /dev/null @@ -1,200 +0,0 @@ -nextflow_workflow { - - name "Test Workflow UTILS_NFVALIDATION_PLUGIN" - script "../main.nf" - workflow "UTILS_NFVALIDATION_PLUGIN" - tag "subworkflows" - tag "subworkflows_nfcore" - tag "plugin/nf-validation" - tag "'plugin/nf-validation'" - tag "utils_nfvalidation_plugin" - tag "subworkflows/utils_nfvalidation_plugin" - - test("Should run nothing") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success } - ) - } - } - - test("Should run help") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with command") { - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = null - post_help_text = null - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } } - ) - } - } - - test("Should run help with extra text") { - - - when { - - params { - monochrome_logs = true - test_data = '' - } - workflow { - """ - help = true - workflow_command = "nextflow run noorg/doesntexist" - pre_help_text = "pre-help-text" - post_help_text = "post-help-text" - validate_params = false - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.success }, - { assert workflow.exitStatus == 0 }, - { assert workflow.stdout.any { it.contains('pre-help-text') } }, - { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, - { assert workflow.stdout.any { it.contains('Input/output options') } }, - { assert workflow.stdout.any { it.contains('--outdir') } }, - { assert workflow.stdout.any { it.contains('post-help-text') } } - ) - } - } - - test("Should validate params") { - - when { - - params { - monochrome_logs = true - test_data = '' - outdir = 1 - } - workflow { - """ - help = false - workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = true - schema_filename = "$moduleTestDir/nextflow_schema.json" - - input[0] = help - input[1] = workflow_command - input[2] = pre_help_text - input[3] = post_help_text - input[4] = validate_params - input[5] = schema_filename - """ - } - } - - then { - assertAll( - { assert workflow.failed }, - { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } - ) - } - } -} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml deleted file mode 100644 index 60b1cfff..00000000 --- a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -subworkflows/utils_nfvalidation_plugin: - - subworkflows/nf-core/utils_nfvalidation_plugin/** diff --git a/tests/nextflow.config b/tests/nextflow.config index 85a76ae6..2d9336b0 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -4,6 +4,8 @@ ======================================================================================== */ +includeConfig !System.getenv('NXF_OFFLINE') ? 'https://raw.githubusercontent.com/nf-core/modules/master/tests/config/test_data.config' : "conf/empty_test_data.config" + // Change some params to test data params { test_data_base = 'https://raw.githubusercontent.com/nf-core/test-datasets/modules' @@ -11,11 +13,6 @@ params { config_profile_name = 'nf-test profile' config_profile_description = 'Configurations for running nf-test' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - input = "${baseDir}/tests/samplesheet.csv" // References for test data @@ -64,9 +61,10 @@ params { } process { - cpus = 2 - memory = 3.GB - time = 2.h -} - -includeConfig 'https://raw.githubusercontent.com/nf-core/modules/master/tests/config/test_data.config' + // Limit resources so that this can run on GitHub Actions + resourceLimits = [ + cpus : 2, + memory: 6.GB, + time : 6.h + ] +} \ No newline at end of file diff --git a/tests/subworkflows/local/bam_cnv_calling/nextflow.config b/tests/subworkflows/local/bam_cnv_calling/nextflow.config deleted file mode 100644 index 3349f1bc..00000000 --- a/tests/subworkflows/local/bam_cnv_calling/nextflow.config +++ /dev/null @@ -1,34 +0,0 @@ -process { - withName: QDNASEQ { - ext.prefix = { "${meta.id}.qdnaseq" } - } - - withName: GAWK { - ext.prefix = { "${meta.id}.qdnaseq.abberations" } - ext.suffix = "bed" - ext.args = "-F '\\t'" - ext.args2 = "'{ if (\$5 >= 0.5 || \$5 <= -0.5 || \$5 == \"Inf\") { print } }'" - } - - withName: WISECONDORX_PREDICT { - ext.prefix = { "${meta.id}.wisecondorx" } - } - - - withName: BEDGOVCF { - ext.prefix = { "${meta.id}.${meta.caller}" } - ext.args = { meta.caller == "qdnaseq" ? "--skip 1" : "--header" } - } - - withName: JASMINESV { - ext.args = {[ - meta.variant_type == "sv" ? "min_support=1" : "", - meta.variant_type == "cnv" ? "min_support=1" : "", - "--allow_intrasample" - ].join(' ').trim()} - } - - withName: BCFTOOLS_SORT { - ext.prefix = { "${meta.id}.${meta.variant_type}" } - } -} diff --git a/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/main.nf.test b/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/main.nf.test index 7e2f0086..a75bec99 100644 --- a/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/main.nf.test +++ b/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/main.nf.test @@ -3,7 +3,6 @@ nextflow_workflow { name "Test Workflow BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER" script "subworkflows/local/bam_repeat_estimation_expansionhunter/main.nf" workflow "BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER" - config "./nextflow.config" tag "subworkflows" tag "subworkflows_local" diff --git a/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/nextflow.config b/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/nextflow.config deleted file mode 100644 index 3f43f7d4..00000000 --- a/tests/subworkflows/local/bam_repeat_estimation_expansionhunter/nextflow.config +++ /dev/null @@ -1,7 +0,0 @@ -process { - withName: BCFTOOLS_ANNOTATE { - ext.args = "-c INFO/REPREF:=INFO/REF --output-type z" - ext.prefix = { "${meta.id}.repeats" } - ext.tabix = true - } -} diff --git a/tests/subworkflows/local/bam_sv_calling/main.nf.test b/tests/subworkflows/local/bam_sv_calling/main.nf.test index fed04ec4..b0fb3e3a 100644 --- a/tests/subworkflows/local/bam_sv_calling/main.nf.test +++ b/tests/subworkflows/local/bam_sv_calling/main.nf.test @@ -10,8 +10,6 @@ nextflow_workflow { options "-stub" - config "./nextflow.config" - test("homo_sapiens - manta") { when { diff --git a/tests/subworkflows/local/bam_sv_calling/nextflow.config b/tests/subworkflows/local/bam_sv_calling/nextflow.config deleted file mode 100644 index c0f34938..00000000 --- a/tests/subworkflows/local/bam_sv_calling/nextflow.config +++ /dev/null @@ -1,41 +0,0 @@ -process { - withName: GAWK { - ext.suffix = "bed" - ext.prefix = "contigs" - ext.args2 = '\'BEGIN {FS="\t"}; \$1 ~ /^chr[1-9XY][1-9]?\$/ {print \$1 FS "0" FS \$2 }\'' - } - - withName: VIOLA { - ext.prefix = { "${meta.id}.viola" } - ext.args = { meta.read_length ? "--read_length ${meta.read_length}" : ""} - } - - withName: JASMINESV { - ext.args = {[ - meta.variant_type == "sv" ? "min_support=1" : "", - meta.variant_type == "cnv" ? "min_support=1" : "", - "--allow_intrasample" - ].join(' ').trim()} - } - - withName: BCFTOOLS_REHEADER { - ext.prefix = { "${meta.id}_reheadered" } - ext.args2 = "--output-type z" - } - - withName: ".*:BAM_VARIANT_CALLING_GRIDSS:BCFTOOLS_SORT\$" { - ext.prefix = { "${meta.id}.gridss" } - } - - withName: ".*:BAM_VARIANT_CALLING_SMOOVE:BCFTOOLS_SORT\$" { - ext.prefix = { "${meta.id}.smoove" } - } - - withName: ".*:BAM_VARIANT_CALLING_MANTA:MANTA_CONVERTINVERSION\$" { - ext.prefix = { "${meta.id}.manta" } - } - - withName: ".*:BAM_VARIANT_CALLING_DELLY:DELLY_CALL\$" { - ext.prefix = { "${meta.id}.delly" } - } -} diff --git a/tests/subworkflows/local/bam_variant_calling_delly/main.nf.test b/tests/subworkflows/local/bam_variant_calling_delly/main.nf.test index 43c983b3..89771a89 100644 --- a/tests/subworkflows/local/bam_variant_calling_delly/main.nf.test +++ b/tests/subworkflows/local/bam_variant_calling_delly/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "bam_variant_calling_delly" - config "./nextflow.config" - test("homo_sapiens") { when { diff --git a/tests/subworkflows/local/bam_variant_calling_delly/nextflow.config b/tests/subworkflows/local/bam_variant_calling_delly/nextflow.config deleted file mode 100644 index 46eb005c..00000000 --- a/tests/subworkflows/local/bam_variant_calling_delly/nextflow.config +++ /dev/null @@ -1,7 +0,0 @@ -process { - withName: DELLY_CALL { - ext.args = "--svtype ALL" - ext.suffix = "vcf" - ext.prefix = { "${meta.id}.delly" } - } -} diff --git a/tests/subworkflows/local/bam_variant_calling_gridss/main.nf.test.disabled b/tests/subworkflows/local/bam_variant_calling_gridss/main.nf.test.disabled index 3d44fde1..40cd074e 100644 --- a/tests/subworkflows/local/bam_variant_calling_gridss/main.nf.test.disabled +++ b/tests/subworkflows/local/bam_variant_calling_gridss/main.nf.test.disabled @@ -8,8 +8,6 @@ // tag "subworkflows_local" // tag "bam_variant_calling_gridss" -// config "./nextflow.config" - // test("homo_sapiens") { // setup { diff --git a/tests/subworkflows/local/bam_variant_calling_gridss/nextflow.config b/tests/subworkflows/local/bam_variant_calling_gridss/nextflow.config deleted file mode 100644 index 32f86102..00000000 --- a/tests/subworkflows/local/bam_variant_calling_gridss/nextflow.config +++ /dev/null @@ -1,18 +0,0 @@ -process { - withName: GRIDSS_GRIDSS { - ext.args = { [ - "--steps preprocess,assemble,call", - "--otherjvmheap ${task.memory.toGiga() > 10 ? 4 : 2}g", - "--jvmheap ${task.memory.toGiga() > 10 ? task.memory.toGiga() - 8 : 2}g" - ].join(" ") } - } - - withName: VIOLA { - ext.prefix = { "${meta.id}.viola" } - ext.args = { meta.read_length ? "--read_length ${meta.read_length}" : ""} - } - - withName: BCFTOOLS_SORT { - ext.prefix = { "${meta.id}.gridss" } - } -} diff --git a/tests/subworkflows/local/bam_variant_calling_manta/main.nf.test b/tests/subworkflows/local/bam_variant_calling_manta/main.nf.test index 5d0d9240..c155318d 100644 --- a/tests/subworkflows/local/bam_variant_calling_manta/main.nf.test +++ b/tests/subworkflows/local/bam_variant_calling_manta/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "bam_variant_calling_manta" - config "./nextflow.config" - test("homo_sapiens") { when { diff --git a/tests/subworkflows/local/bam_variant_calling_manta/nextflow.config b/tests/subworkflows/local/bam_variant_calling_manta/nextflow.config deleted file mode 100644 index 6297a2d5..00000000 --- a/tests/subworkflows/local/bam_variant_calling_manta/nextflow.config +++ /dev/null @@ -1,11 +0,0 @@ -process { - withName: GAWK { - ext.suffix = "bed" - ext.prefix = "contigs" - ext.args2 = '\'BEGIN {FS="\t"}; \$1 ~ /^chr[1-9XY][1-9]?\$/ {print \$1 FS "0" FS \$2 }\'' - } - - withName: MANTA_CONVERTINVERSION { - ext.prefix = { "${meta.id}.manta" } - } -} diff --git a/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test b/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test index f017b86a..4a24b3fa 100644 --- a/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test +++ b/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "bam_variant_calling_qdnaseq" - config "./nextflow.config" - test("homo_sapiens") { when { diff --git a/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test.snap b/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test.snap index ee62291a..f9b2c077 100644 --- a/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test.snap +++ b/tests/subworkflows/local/bam_variant_calling_qdnaseq/main.nf.test.snap @@ -27,7 +27,7 @@ "sex": "male", "caller": "qdnaseq" }, - "test.bed:md5,19be72c3f56297070c6070ded3419dee" + "test.qdnaseq.bed:md5,19be72c3f56297070c6070ded3419dee" ], [ { @@ -36,7 +36,7 @@ "sex": "female", "caller": "qdnaseq" }, - "test2.bed:md5,fed14c95db6bed194f40f262d68399d1" + "test2.qdnaseq.bed:md5,fed14c95db6bed194f40f262d68399d1" ] ], 10, diff --git a/tests/subworkflows/local/bam_variant_calling_qdnaseq/nextflow.config b/tests/subworkflows/local/bam_variant_calling_qdnaseq/nextflow.config deleted file mode 100644 index e9791020..00000000 --- a/tests/subworkflows/local/bam_variant_calling_qdnaseq/nextflow.config +++ /dev/null @@ -1,13 +0,0 @@ -process { - withName: GAWK { - ext.prefix = { "${meta.id}.qdnaseq.abberations" } - ext.suffix = "bed" - ext.args = "-F '\\t'" - ext.args2 = "'{ if (\$5 >= 0.5 || \$5 <= -0.5 || \$5 == \"Inf\") { print } }'" - } - - withName: BEDGOVCF { - ext.prefix = { "${meta.id}.${meta.caller}" } - ext.args = "--skip 1" - } -} diff --git a/tests/subworkflows/local/bam_variant_calling_wisecondorx/main.nf.test.snap b/tests/subworkflows/local/bam_variant_calling_wisecondorx/main.nf.test.snap index d5584ace..5d870493 100644 --- a/tests/subworkflows/local/bam_variant_calling_wisecondorx/main.nf.test.snap +++ b/tests/subworkflows/local/bam_variant_calling_wisecondorx/main.nf.test.snap @@ -76,7 +76,7 @@ "sample": "test", "sex": "male" }, - "test.vcf.gz" + "test.wisecondorx.vcf.gz" ], 4, 0 diff --git a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test b/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test index 2c0b469d..6fa022d0 100644 --- a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test +++ b/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test @@ -9,7 +9,6 @@ nextflow_workflow { tag "vcf_annotate_vep_annotsv_vcfanno" options "-stub" - config "./nextflow.config" test("homo_sapiens - sv, no_filter") { diff --git a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test.snap b/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test.snap index fc707522..69f24fa6 100644 --- a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test.snap +++ b/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/main.nf.test.snap @@ -14,8 +14,8 @@ ], "variant_type": "sv" }, - "test.sv.vcf.gz", - "test.sv.vcf.gz.tbi" + "test.sv.annotated.vcf.gz", + "test.sv.annotated.vcf.gz.tbi" ] ], [ @@ -45,8 +45,8 @@ ], "variant_type": "cnv" }, - "test.cnv.vcf.gz", - "test.cnv.vcf.gz.tbi" + "test.cnv.annotated.vcf.gz", + "test.cnv.annotated.vcf.gz.tbi" ], [ { @@ -60,8 +60,8 @@ ], "variant_type": "sv" }, - "test.sv.vcf.gz", - "test.sv.vcf.gz.tbi" + "test.sv.annotated.vcf.gz", + "test.sv.annotated.vcf.gz.tbi" ] ], [ @@ -92,8 +92,8 @@ ], "variant_type": "cnv" }, - "test.cnv.vcf.gz", - "test.cnv.vcf.gz.tbi" + "test.cnv.annotated.vcf.gz", + "test.cnv.annotated.vcf.gz.tbi" ] ], [ diff --git a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/nextflow.config b/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/nextflow.config deleted file mode 100644 index 963c69ea..00000000 --- a/tests/subworkflows/local/vcf_annotate_vep_annotsv_vcfanno/nextflow.config +++ /dev/null @@ -1,61 +0,0 @@ -process { - - withName: BCFTOOLS_FILTER { - ext.prefix = {"${meta.id}.filter"} - ext.args = "-e 'GT=\"ref\"' --output-type z" - } - - withName: ANNOTSV_ANNOTSV { - ext.args = {[ - "-SVminSize 20", - "-vcf 1", - meta.hpo ? "-hpo ${meta.hpo}" : "" - ].join(" ")} - ext.prefix = {"${meta.id}.annot"} - } - - withName: BCFTOOLS_CONCAT { - ext.prefix = params.annotsv_file_name - ext.args = "--output-type z --naive-force" - } - - withName: ENSEMBLVEP_VEP { - ext.prefix = {"${meta.id}.vep"} - ext.args = {[ - // specify we use VCF files - '--format vcf', - // don't contact external db - '--offline', - // increase buffer_size to speed up analysis - '--buffer_size 100000', - // output format options - '--vcf --compress_output bgzip --force_overwrite', - // co-located variant info - '--check_existing', - // specific options for structural variants - '--overlaps', // TODO define the best configuration for --max_sv_size, --batch_size, --check_svs and --database - // annotations - '--regulatory --pubmed --symbol --hgvsg --hgvs --af_gnomadg --sift s', - // proteins - '--domains --biotype --canonical --mane --ccds --protein --polyphen s --sift s', - ].join(' ').trim()} - } - - withName: VCFANNO { - ext.args = "-permissive-overlap -ends" - } - - if(!params.annotations_filter) { - withName: TABIX_ANNOTATED { - ext.prefix = { "${meta.id}.${meta.variant_type}" } - } - } - - if(params.annotations_filter) { - withName: BCFTOOLS_FILTER_COMMON { - ext.prefix = {"${meta.id}.${meta.variant_type}"} - ext.args = "${params.annotations_filter} --output-type z" - } - } - -} diff --git a/tests/subworkflows/local/vcf_concat_bcftools/main.nf.test b/tests/subworkflows/local/vcf_concat_bcftools/main.nf.test index 46a5577c..cac6f8ee 100644 --- a/tests/subworkflows/local/vcf_concat_bcftools/main.nf.test +++ b/tests/subworkflows/local/vcf_concat_bcftools/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "vcf_concat_bcftools" - config "./nextflow.config" - test("homo_sapiens") { when { diff --git a/tests/subworkflows/local/vcf_concat_bcftools/nextflow.config b/tests/subworkflows/local/vcf_concat_bcftools/nextflow.config deleted file mode 100644 index a73ba928..00000000 --- a/tests/subworkflows/local/vcf_concat_bcftools/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - withName: BCFTOOLS_CONCAT { - ext.args = "--allow-overlaps --output-type z" - } -} diff --git a/tests/subworkflows/local/vcf_merge_callers_jasmine/main.nf.test b/tests/subworkflows/local/vcf_merge_callers_jasmine/main.nf.test index b00456bf..572d22c0 100644 --- a/tests/subworkflows/local/vcf_merge_callers_jasmine/main.nf.test +++ b/tests/subworkflows/local/vcf_merge_callers_jasmine/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "vcf_merge_callers_jasmine" - config "./nextflow.config" - test("homo_sapiens - multiple callers") { when { diff --git a/tests/subworkflows/local/vcf_merge_callers_jasmine/nextflow.config b/tests/subworkflows/local/vcf_merge_callers_jasmine/nextflow.config deleted file mode 100644 index 31baa11e..00000000 --- a/tests/subworkflows/local/vcf_merge_callers_jasmine/nextflow.config +++ /dev/null @@ -1,23 +0,0 @@ -process { - withName: JASMINESV { - ext.args = {[ - meta.variant_type == "sv" ? "min_support=${params.sv_callers_support}" : "", - meta.variant_type == "cnv" ? "min_support=${params.cnv_callers_support}" : "", - "--allow_intrasample" - ].join(' ').trim()} - } - - withName: FIX_CALLERS { - ext.prefix = { "${meta.id}_fixed" } - } - - withName: BCFTOOLS_REHEADER { - ext.prefix = { "${meta.id}_reheadered" } - ext.args2 = "--output-type z" - } - - withName: BCFTOOLS_SORT { - ext.prefix = { "${meta.id}.${meta.variant_type}" } - } - -} diff --git a/tests/subworkflows/local/vcf_merge_family_jasmine/main.nf.test b/tests/subworkflows/local/vcf_merge_family_jasmine/main.nf.test index 95e9e005..7b69cfb0 100644 --- a/tests/subworkflows/local/vcf_merge_family_jasmine/main.nf.test +++ b/tests/subworkflows/local/vcf_merge_family_jasmine/main.nf.test @@ -8,8 +8,6 @@ nextflow_workflow { tag "subworkflows_local" tag "vcf_merge_family_jasmine" - config "./nextflow.config" - test("homo_sapiens - one family, variant types") { when { diff --git a/tests/subworkflows/local/vcf_merge_family_jasmine/nextflow.config b/tests/subworkflows/local/vcf_merge_family_jasmine/nextflow.config deleted file mode 100644 index 7ebe798c..00000000 --- a/tests/subworkflows/local/vcf_merge_family_jasmine/nextflow.config +++ /dev/null @@ -1,18 +0,0 @@ -process { - withName: JASMINESV { - ext.args = "--output_genotypes" - } - - withName: FIX_CALLERS { - ext.prefix = { "${meta.id}.callers-corrected" } - } - - withName: BCFTOOLS_CONSENSUS_REHEADER { - ext.prefix = { "${meta.id}.reheadered" } - } - - withName: BCFTOOLS_SORT { - ext.args = "--output-type z" - } - -} diff --git a/workflows/structural.nf b/workflows/structural.nf index ebdd77d1..260bc969 100644 --- a/workflows/structural.nf +++ b/workflows/structural.nf @@ -6,7 +6,7 @@ import java.util.Scanner ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryMap } from 'plugin/nf-validation' +include { paramsSummaryMap } from 'plugin/nf-schema' include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_structural_pipeline' @@ -87,7 +87,7 @@ workflow STRUCTURAL { concat_output // Concatenate all output files // value inputs - callers // All callers to be used + input_callers // All callers to be used sv_callers_support // How many SV callers are needed to support the variant cnv_callers_support // How many CNV callers are needed to support the variant genome // The genome to use @@ -113,26 +113,25 @@ workflow STRUCTURAL { // // When making changes here, make sure to also update the following files: conf/modules.config - def List svCallers = ["delly", "manta", "smoove"] //, "gridss" - def List cnvCallers = ["qdnaseq", "wisecondorx"] - def List repeatsCallers = ["expansionhunter"] + def svCallers = ["delly", "manta", "smoove"] //, "gridss" + def cnvCallers = ["qdnaseq", "wisecondorx"] + def repeatsCallers = ["expansionhunter"] - def List allCallers = svCallers + cnvCallers + repeatsCallers - def List annotationCallers = svCallers + cnvCallers + def allCallers = svCallers + cnvCallers + repeatsCallers + def annotationCallers = svCallers + cnvCallers // Callers that need the sex - def List sexCallers = ["expansionhunter", "qdnaseq"] - - def lower_cased_callers = callers.toLowerCase() - def callers = lower_cased_callers.tokenize(",").collect { - if(it == "all") {return allCallers} - if(it == "sv") {return svCallers} - if(it == "cnv") {return cnvCallers} - if(it == "rre") {return repeatsCallers} - return it + def sexCallers = ["expansionhunter", "qdnaseq"] + + def callers = input_callers.toLowerCase().tokenize(",").collect { caller -> + if(caller == "all") {return allCallers} + if(caller == "sv") {return svCallers} + if(caller == "cnv") {return cnvCallers} + if(caller == "rre") {return repeatsCallers} + return caller }.flatten() - for (caller in callers) { + callers.each { caller -> if(!(caller in allCallers)) { error("The caller '${caller}' is not supported please specify a comma delimited list with on or more of the following callers: ${allCallers}".toString()) } } @@ -160,20 +159,20 @@ workflow STRUCTURAL { // Create input channels from parameters // - ch_fasta = Channel.fromPath(fasta).map{[[id:'fasta'], it]}.collect() - ch_annotsv_candidate_genes = annotsv_candidate_genes ? Channel.fromPath(annotsv_candidate_genes).map{[[], it]}.collect() : [[],[]] - ch_annotsv_gene_transcripts = annotsv_gene_transcripts ? Channel.fromPath(annotsv_gene_transcripts).map{[[], it]}.collect() : [[],[]] + ch_fasta = Channel.fromPath(fasta).collect { fasta_file -> [[id:'fasta'], fasta_file ] } + ch_annotsv_candidate_genes = annotsv_candidate_genes ? Channel.fromPath(annotsv_candidate_genes).collect { genes_file -> [[], genes_file] } : [[],[]] + ch_annotsv_gene_transcripts = annotsv_gene_transcripts ? Channel.fromPath(annotsv_gene_transcripts).collect { transcripts_file -> [[], transcripts_file] } : [[],[]] ch_vcfanno_lua = vcfanno_lua ? Channel.fromPath(vcfanno_lua).collect() : [] - ch_catalog = expansionhunter_catalog ? Channel.fromPath(expansionhunter_catalog).map{[[id:'catalog'], it]}.collect() : [[],[]] - ch_qdnaseq_male = qdnaseq_male ? Channel.fromPath(qdnaseq_male).map{[[id:'qdnaseq'], it]}.collect() : [[],[]] - ch_qdnaseq_female = qdnaseq_female ? Channel.fromPath(qdnaseq_female).map{[[id:'qdnaseq'], it]}.collect() : [[],[]] - ch_wisecondorx_reference = wisecondorx_reference ? Channel.fromPath(wisecondorx_reference).map{[[id:'wisecondorx'], it]}.collect() : [[],[]] - ch_blacklist = blacklist ? Channel.fromPath(blacklist).map{[[id:'blacklist'], it]}.collect() : [[],[]] + ch_catalog = expansionhunter_catalog ? Channel.fromPath(expansionhunter_catalog).collect { catalog_file -> [[id:'catalog'], catalog_file] } : [[],[]] + ch_qdnaseq_male = qdnaseq_male ? Channel.fromPath(qdnaseq_male).collect { qdnaseq_file -> [[id:'qdnaseq_male'], qdnaseq_file] } : [[],[]] + ch_qdnaseq_female = qdnaseq_female ? Channel.fromPath(qdnaseq_female).collect { qdnaseq_file -> [[id:'qdnaseq_female'], qdnaseq_file] } : [[],[]] + ch_wisecondorx_reference = wisecondorx_reference ? Channel.fromPath(wisecondorx_reference).collect { wcx_file -> [[id:'wisecondorx'], wcx_file] } : [[],[]] + ch_blacklist = blacklist ? Channel.fromPath(blacklist).collect { blacklist_file -> [[id:'blacklist'], blacklist_file] } : [[],[]] ch_manta_config = manta_config ? Channel.fromPath(manta_config).collect() : [[]] ch_svync_configs = svync_dir ? Channel.fromPath("${svync_dir}/*.yaml", checkIfExists:true).collect() : [] ch_bedgovcf_configs = bedgovcf_dir ? Channel.fromPath("${bedgovcf_dir}/*.yaml", checkIfExists:true).collect() : [] - val_vcfanno_resources = vcfanno_resources ? vcfanno_resources.split(",").collect{file(it, checkIfExists:true)}.flatten() : [] + val_vcfanno_resources = vcfanno_resources ? vcfanno_resources.split(",").collect { resource_file -> file(resource_file, checkIfExists:true) }.flatten() : [] val_default_vcfanno_tomls = vcfanno_default_dir ? files("${vcfanno_default_dir}/*.toml", checkIfExists:true) : [] val_vcfanno_toml = vcfanno_toml ? file(vcfanno_toml, checkIfExists:true) : [] @@ -189,10 +188,10 @@ workflow STRUCTURAL { ) ch_versions = ch_versions.mix(SAMTOOLS_FAIDX.out.versions) - ch_fai = SAMTOOLS_FAIDX.out.fai.map{[[id:'fai'], it]}.collect() + ch_fai = SAMTOOLS_FAIDX.out.fai.collect { fai_file -> [[id:'fai'], fai_file] } } else { - ch_fai = Channel.fromPath(fai).map{[[id:"fai"],it]}.collect() + ch_fai = Channel.fromPath(fai).collect { fai_file -> [[id:'fai'], fai_file] } } // if(!bwa && "gridss" in callers){ @@ -227,12 +226,11 @@ workflow STRUCTURAL { ch_versions = ch_versions.mix(ANNOTSV_INSTALLANNOTATIONS.out.versions) ANNOTSV_INSTALLANNOTATIONS.out.annotations - .map { [[id:"annotsv"], it] } - .collect() + .collect { annotations -> [[id:"annotsv_annotations"], annotations] } .set { ch_annotsv_annotations } } else if(annotate && callers.intersect(annotationCallers)) { - ch_annotsv_annotations_input = Channel.fromPath(annotsv_annotations).map{[[id:"annotsv_annotations"], it]}.collect() + ch_annotsv_annotations_input = Channel.fromPath(annotsv_annotations).collect { annotations -> [[id:"annotsv_annotations"], annotations] } if(annotsv_annotations.endsWith(".tar.gz")){ UNTAR_ANNOTSV( ch_annotsv_annotations_input @@ -243,7 +241,7 @@ workflow STRUCTURAL { .collect() .set { ch_annotsv_annotations } } else { - ch_annotsv_annotations = Channel.fromPath(annotsv_annotations).map{[[id:"annotsv_annotations"], it]}.collect() + ch_annotsv_annotations = Channel.fromPath(annotsv_annotations).collect { annotations -> [[id:"annotsv_annotations"], annotations] } } } else { @@ -256,7 +254,7 @@ workflow STRUCTURAL { ) ch_versions = ch_versions.mix(ENSEMBLVEP_DOWNLOAD.out.versions) - ch_vep_cache = ENSEMBLVEP_DOWNLOAD.out.cache.map{it[1]}.collect() + ch_vep_cache = ENSEMBLVEP_DOWNLOAD.out.cache.collect { annotations -> annotations[1] } } else if (vep_cache && annotate && callers.intersect(annotationCallers)) { ch_vep_cache = Channel.fromPath(vep_cache).collect() @@ -270,17 +268,21 @@ workflow STRUCTURAL { // BAM_PREPARE_SAMTOOLS( - ch_samplesheet.map{ it.subList(0, 3) }, + ch_samplesheet.map { meta, cram, crai, small_variants -> + [ meta, cram, crai ] + }, ch_fasta, ch_fai ) ch_versions = ch_versions.mix(BAM_PREPARE_SAMTOOLS.out.versions) ch_samplesheet - .map{ [it[0]] + it.subList(3, it.size()) } + .map { meta, cram, crai, small_variants -> + [ meta, small_variants ] + } .groupTuple() - .map { - [ it[0] ] + it.subList(1, it.size()).collect { it.find { it != [] } ?: [] } + .map { meta, small_variants -> + [ meta, small_variants.find { small_variant -> small_variant != [] } ?: [] ] } .set { ch_deduplicated } @@ -292,16 +294,17 @@ workflow STRUCTURAL { // Determine the gender if needed // + ch_input_sex = Channel.empty() if(callers.intersect(sexCallers)) { ch_input_no_sex - .branch { - sex: it[0].sex - no_sex: !it[0].sex + .branch { meta, cram, crai, small_variants -> + sex: meta.sex + no_sex: !meta.sex } .set { ch_samplegender_input } NGSBITS_SAMPLEGENDER( - ch_samplegender_input.no_sex.map{ it.subList(0, 3) }, + ch_samplegender_input.no_sex, ch_fasta, ch_fai, "xy" @@ -310,9 +313,9 @@ workflow STRUCTURAL { NGSBITS_SAMPLEGENDER.out.tsv .join(ch_samplegender_input.no_sex, failOnDuplicate:true, failOnMismatch:true) - .map { - new_meta = it[0] + [sex:get_sex(it[1], it[0].sample)] - return [ new_meta ] + it.subList(2, it.size()) + .map { meta, tsv, cram, crai -> + def new_meta = meta + [sex:get_sex(tsv, meta.sample)] + return [ new_meta, cram, crai ] } .mix(ch_samplegender_input.sex) .set { ch_input_sex } @@ -321,10 +324,10 @@ workflow STRUCTURAL { } ch_input_sex - .multiMap({ meta, cram, crai, small_variants -> + .multiMap { meta, cram, crai, small_variants -> crams: [ meta, cram, crai ] small_variants: [ meta, small_variants ] - }) + } .set { ch_inputs } // @@ -333,7 +336,7 @@ workflow STRUCTURAL { if(sv_callers_to_use){ - count_types++ + count_types += 1 variant_types.add("sv") BAM_SV_CALLING( @@ -357,7 +360,7 @@ workflow STRUCTURAL { if(cnv_callers_to_use){ - count_types++ + count_types += 1 variant_types.add("cnv") BAM_CNV_CALLING( @@ -414,7 +417,7 @@ workflow STRUCTURAL { if(callers.intersect(repeatsCallers)){ - count_types++ + count_types += 1 BAM_REPEAT_ESTIMATION_EXPANSIONHUNTER( ch_inputs.crams, @@ -431,8 +434,8 @@ workflow STRUCTURAL { // Concatenate the VCF files from different types of analysis // + ch_concat_vcfs = Channel.empty() if(count_types > 1 && concat_output) { - ch_outputs .map { meta, vcf, tbi -> def new_meta = meta - meta.subMap("variant_type") @@ -448,10 +451,8 @@ workflow STRUCTURAL { VCF_CONCAT_BCFTOOLS.out.vcfs .set { ch_concat_vcfs } - } else { - ch_outputs - .set { ch_concat_vcfs } + ch_outputs.set { ch_concat_vcfs } } // @@ -486,7 +487,9 @@ workflow STRUCTURAL { ch_multiqc_files.collect(), ch_multiqc_config.toList(), ch_multiqc_custom_config.toList(), - ch_multiqc_logo.toList() + ch_multiqc_logo.toList(), + [], + [] ) emit: @@ -506,10 +509,11 @@ def get_sex(tsv, id) { return "female" log.warn("STUB: Couldn't define the sex of sample ${id}. Defaulting to female. (Specify the sex in the samplesheet to avoid this warning.)") } - split_tsv = tsv.splitCsv(sep:"\t", header:true, strip:true) - sex = split_tsv[0].gender - if(sex != "male" || sex != "female") { - error("Couldn't define the sex of sample ${id}. Specify the sex in the samplesheet to avoid this error and resume the pipeline.") + def split_tsv = tsv.splitCsv(sep:"\t", header:true, strip:true) + def sex = split_tsv[0].gender + if(sex == "others") { + sex = "female" + log.warn("Couldn't define the sex of sample ${id}. Defaulting to female. (Specify the sex in the samplesheet to avoid this warning.)") } return sex }