From f3085fb101e621989f3e0f0afd034ff3b6436273 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 13:07:08 -0400 Subject: [PATCH 1/6] feat: create module for picard samtofastq --- modules/CCBR/picard/samtofastq/main.nf | 40 +++++++++++++++++ modules/CCBR/picard/samtofastq/meta.yml | 44 +++++++++++++++++++ tests/config/pytest_modules.yml | 4 ++ tests/modules/CCBR/picard/samtofastq/main.nf | 26 +++++++++++ .../CCBR/picard/samtofastq/nextflow.config | 7 +++ 5 files changed, 121 insertions(+) create mode 100644 modules/CCBR/picard/samtofastq/main.nf create mode 100644 modules/CCBR/picard/samtofastq/meta.yml create mode 100644 tests/modules/CCBR/picard/samtofastq/main.nf create mode 100644 tests/modules/CCBR/picard/samtofastq/nextflow.config diff --git a/modules/CCBR/picard/samtofastq/main.nf b/modules/CCBR/picard/samtofastq/main.nf new file mode 100644 index 0000000..63c7db7 --- /dev/null +++ b/modules/CCBR/picard/samtofastq/main.nf @@ -0,0 +1,40 @@ +process PICARD_SAMTOFASTQ { + tag { meta.id } + label 'process_medium' + + container 'nciccbr/ccbr_picard_2.27.5:v1' + + input: + tuple val(meta), path(bam) + + output: + tuple val(meta), path("*.fastq"), emit: reads + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def output = meta.single_end ? "--FASTQ ${prefix}.fastq.gz" : "--FASTQ ${prefix}_1.fastq.gz --SECOND_END_FASTQ ${prefix}_2.fastq.gz" + + if (!task.memory) { + log.warn '[Picard SamToFastq] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' + } + def avail_mem = task.memory ? (task.memory.mega*0.8).intValue() : 3072 + + """ + picard \\ + -Xmx${avail_mem}M \\ + SamToFastq \\ + ${args} \\ + --INPUT ${bam} \\ + ${output} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + picard: \$(picard SamToFastq --version 2>&1 | grep -o 'Version:.*' | cut -f2- -d:) + END_VERSIONS + """ +} diff --git a/modules/CCBR/picard/samtofastq/meta.yml b/modules/CCBR/picard/samtofastq/meta.yml new file mode 100644 index 0000000..4d4b8e3 --- /dev/null +++ b/modules/CCBR/picard/samtofastq/meta.yml @@ -0,0 +1,44 @@ +name: "picard_samtofastq" +description: Converts a BAM or SAM file to fastq. +keywords: + - fastq + - bam + - sam +tools: + - picard: + description: | + A set of command line tools (in Java) for manipulating high-throughput sequencing (HTS) + data and formats such as SAM/BAM/CRAM and VCF. + homepage: https://broadinstitute.github.io/picard/ + documentation: https://gatk.broadinstitute.org/hc/en-us/articles/360036510672-FastqToSam-Picard- + tool_dev_url: https://github.com/broadinstitute/picard + licence: ["MIT"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - bam: + type: file + description: | + SAM or BAM file +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + # + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - fastq: + type: file + description: fastq files + pattern: "*.{fastq}" +authors: + - "@kelly-sovacool" +maintainers: + - "@kelly-sovacool" diff --git a/tests/config/pytest_modules.yml b/tests/config/pytest_modules.yml index 6b9d095..ad02019 100644 --- a/tests/config/pytest_modules.yml +++ b/tests/config/pytest_modules.yml @@ -18,6 +18,10 @@ khmer/uniquekmers: - modules/CCBR/khmer/uniquekmers/** - tests/CCBR/khmer/uniquekmers/** +picard/samtofastq: + - modules/CCBR/picard/samtofastq/** + - tests/modules/CCBR/picard/samtofastq/** + samtools/filteraligned: - modules/CCBR/samtools/filteraligned/** - tests/modules/CCBR/samtools/filteraligned/** diff --git a/tests/modules/CCBR/picard/samtofastq/main.nf b/tests/modules/CCBR/picard/samtofastq/main.nf new file mode 100644 index 0000000..ea6d348 --- /dev/null +++ b/tests/modules/CCBR/picard/samtofastq/main.nf @@ -0,0 +1,26 @@ +#!/usr/bin/env nextflow + +nextflow.enable.dsl = 2 + +include { PICARD_SAMTOFASTQ } from '../../../../../modules/nf-core/picard/samtofastq/main.nf' + +workflow test_picard_samtofastq_single { + input = [ [ id:'test', single_end: true ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) ] + ] + PICARD_SAMTOFASTQ ( input ) +} + +workflow test_picard_samtofastq_paired { + input = [ [ id:'test', single_end: false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] + ] + PICARD_SAMTOFASTQ ( input ) +} + +workflow test_picard_samtofastq_paired_keep_unpaired { + input = [ [ id:'test', single_end: false ], // meta map + [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] + ] + PICARD_SAMTOFASTQ ( input ) +} diff --git a/tests/modules/CCBR/picard/samtofastq/nextflow.config b/tests/modules/CCBR/picard/samtofastq/nextflow.config new file mode 100644 index 0000000..66840e0 --- /dev/null +++ b/tests/modules/CCBR/picard/samtofastq/nextflow.config @@ -0,0 +1,7 @@ +process { + + publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } + withName: "test_picard_samtofastq_paired_keep_unpaired" { + ext.args = "--UNPAIRED_FASTQ test.unpaired.fastq" + } +} From d8d5c6f7e163d0318f79d6b1fdee480422e7f76e Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 13:09:30 -0400 Subject: [PATCH 2/6] docs: new picard/samtofastq module --- CHANGELOG.md | 1 + README.md | 3 ++- modules/CCBR/picard/samtofastq/meta.yml | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e6b096..7006129 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,5 +7,6 @@ - custom/bam_to_fastq (#14) - cutadapt (#11) - khmer/uniquekmers (#7) +- picard/samtofastq (#21) - samtools/filteraligned (#13,#20) - also runs samtools sort & outputs index in bai format. (#12) diff --git a/README.md b/README.md index d0f936f..d43ea17 100644 --- a/README.md +++ b/README.md @@ -38,5 +38,6 @@ Many of the modules and subworkflows in this project reuse and adapt code from [ In those cases, credit is noted in the `meta.yml` file of the module/subworkflow and also listed here: - [bwa](modules/CCBR/bwa) adapts the [nf-core bwa module](https://github.com/nf-core/chipseq/tree/51eba00b32885c4d0bec60db3cb0a45eb61e34c5/modules/nf-core/modules/bwa) -- [cutadapt](modules/CCBR/cutadapt) adapts the [nf-core cutadapt module](https://github.com/nf-core/modules/tree/master/modules/nf-core/cutadapt) +- [cutadapt](modules/CCBR/cutadapt) adapts the [nf-core cutadapt module](https://github.com/nf-core/modules/tree/ef007b1ce5316506b8c27c3e7a62482409c6153c/modules/nf-core/cutadapt) - [khmer](modules/CCBR/khmer) adapts the [nf-core khmer module](https://github.com/nf-core/modules/tree/b48a1efc8e067502e1a9bafbac788c1e0abdfc6a/modules/nf-core/khmer) +- [picard/samtofastq] adapts the [nf-core gatk4 samtofastq module](https://github.com/nf-core/modules/tree/ef007b1ce5316506b8c27c3e7a62482409c6153c/modules/nf-core/gatk4/samtofastq) diff --git a/modules/CCBR/picard/samtofastq/meta.yml b/modules/CCBR/picard/samtofastq/meta.yml index 4d4b8e3..b99b4c0 100644 --- a/modules/CCBR/picard/samtofastq/meta.yml +++ b/modules/CCBR/picard/samtofastq/meta.yml @@ -1,5 +1,5 @@ name: "picard_samtofastq" -description: Converts a BAM or SAM file to fastq. +description: Converts a BAM or SAM file to fastq. Adapted from the nf-core gatk4 samtofastq module. keywords: - fastq - bam From eda870358696cf14175ebb898db584f14329e3aa Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 14:07:51 -0400 Subject: [PATCH 3/6] feat: keep unpaired fastq; write stub --- modules/CCBR/picard/samtofastq/main.nf | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/CCBR/picard/samtofastq/main.nf b/modules/CCBR/picard/samtofastq/main.nf index 63c7db7..7d538dc 100644 --- a/modules/CCBR/picard/samtofastq/main.nf +++ b/modules/CCBR/picard/samtofastq/main.nf @@ -8,7 +8,7 @@ process PICARD_SAMTOFASTQ { tuple val(meta), path(bam) output: - tuple val(meta), path("*.fastq"), emit: reads + tuple val(meta), path("*.fastq.gz"), emit: reads path "versions.yml" , emit: versions when: @@ -17,7 +17,7 @@ process PICARD_SAMTOFASTQ { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def output = meta.single_end ? "--FASTQ ${prefix}.fastq.gz" : "--FASTQ ${prefix}_1.fastq.gz --SECOND_END_FASTQ ${prefix}_2.fastq.gz" + def output = meta.single_end ? "--FASTQ ${prefix}.fastq" : "--FASTQ ${prefix}_1.fastq --SECOND_END_FASTQ ${prefix}_2.fastq --UNPAIRED_FASTQ ${prefix}.unpaired.fastq" if (!task.memory) { log.warn '[Picard SamToFastq] Available memory not known - defaulting to 3GB. Specify process memory requirements to change this.' @@ -32,9 +32,17 @@ process PICARD_SAMTOFASTQ { --INPUT ${bam} \\ ${output} + pigz -p ${task.cpus} *.fastq + cat <<-END_VERSIONS > versions.yml "${task.process}": picard: \$(picard SamToFastq --version 2>&1 | grep -o 'Version:.*' | cut -f2- -d:) END_VERSIONS """ + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.fastq.gz + """ } From 92a7a0b242b09b719bb3fa479b40c26e78ed9512 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 14:08:24 -0400 Subject: [PATCH 4/6] test: new tests for picard/samtofastq --- tests/modules/CCBR/picard/samtofastq/main.nf | 9 +---- .../CCBR/picard/samtofastq/nextflow.config | 3 -- tests/modules/CCBR/picard/samtofastq/test.yml | 34 +++++++++++++++++++ 3 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 tests/modules/CCBR/picard/samtofastq/test.yml diff --git a/tests/modules/CCBR/picard/samtofastq/main.nf b/tests/modules/CCBR/picard/samtofastq/main.nf index ea6d348..547fb66 100644 --- a/tests/modules/CCBR/picard/samtofastq/main.nf +++ b/tests/modules/CCBR/picard/samtofastq/main.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -include { PICARD_SAMTOFASTQ } from '../../../../../modules/nf-core/picard/samtofastq/main.nf' +include { PICARD_SAMTOFASTQ } from '../../../../../modules/CCBR/picard/samtofastq/main.nf' workflow test_picard_samtofastq_single { input = [ [ id:'test', single_end: true ], // meta map @@ -17,10 +17,3 @@ workflow test_picard_samtofastq_paired { ] PICARD_SAMTOFASTQ ( input ) } - -workflow test_picard_samtofastq_paired_keep_unpaired { - input = [ [ id:'test', single_end: false ], // meta map - [ file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) ] - ] - PICARD_SAMTOFASTQ ( input ) -} diff --git a/tests/modules/CCBR/picard/samtofastq/nextflow.config b/tests/modules/CCBR/picard/samtofastq/nextflow.config index 66840e0..8341169 100644 --- a/tests/modules/CCBR/picard/samtofastq/nextflow.config +++ b/tests/modules/CCBR/picard/samtofastq/nextflow.config @@ -1,7 +1,4 @@ process { publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - withName: "test_picard_samtofastq_paired_keep_unpaired" { - ext.args = "--UNPAIRED_FASTQ test.unpaired.fastq" - } } diff --git a/tests/modules/CCBR/picard/samtofastq/test.yml b/tests/modules/CCBR/picard/samtofastq/test.yml new file mode 100644 index 0000000..30a59a8 --- /dev/null +++ b/tests/modules/CCBR/picard/samtofastq/test.yml @@ -0,0 +1,34 @@ +- name: picard samtofastq test_picard_samtofastq_single + command: nextflow run ./tests/modules/CCBR/picard/samtofastq -entry test_picard_samtofastq_single -c ./tests/config/nextflow.config + tags: + - picard/samtofastq + - picard + files: + - path: output/picard/test.fastq.gz + contains: + - "@ERR5069949.2151832" + - path: output/picard/versions.yml + +- name: picard samtofastq test_picard_samtofastq_paired + command: nextflow run ./tests/modules/CCBR/picard/samtofastq -entry test_picard_samtofastq_paired -c ./tests/config/nextflow.config + tags: + - picard/samtofastq + - picard + files: + - path: output/picard/test_1.fastq.gz + contains: + - "@ERR5069949.2151832/1" + - path: output/picard/test_2.fastq.gz + contains: + - "@ERR5069949.2151832/2" + - path: output/picard/test.unpaired.fastq.gz + - path: output/picard/versions.yml + +- name: picard samtofastq test_picard_samtofastq_single stub + command: nextflow run ./tests/modules/CCBR/picard/samtofastq -entry test_picard_samtofastq_single -c ./tests/config/nextflow.config -stub + tags: + - picard/samtofastq + - picard + files: + - path: output/picard/test.fastq.gz + - path: output/picard/versions.yml From ec991fc8d2d12902f7f1c657a8d75fcbe3a57cd7 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 14:19:08 -0400 Subject: [PATCH 5/6] docs: link to picard module --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d43ea17..9c213eb 100644 --- a/README.md +++ b/README.md @@ -40,4 +40,4 @@ In those cases, credit is noted in the `meta.yml` file of the module/subworkflow - [bwa](modules/CCBR/bwa) adapts the [nf-core bwa module](https://github.com/nf-core/chipseq/tree/51eba00b32885c4d0bec60db3cb0a45eb61e34c5/modules/nf-core/modules/bwa) - [cutadapt](modules/CCBR/cutadapt) adapts the [nf-core cutadapt module](https://github.com/nf-core/modules/tree/ef007b1ce5316506b8c27c3e7a62482409c6153c/modules/nf-core/cutadapt) - [khmer](modules/CCBR/khmer) adapts the [nf-core khmer module](https://github.com/nf-core/modules/tree/b48a1efc8e067502e1a9bafbac788c1e0abdfc6a/modules/nf-core/khmer) -- [picard/samtofastq] adapts the [nf-core gatk4 samtofastq module](https://github.com/nf-core/modules/tree/ef007b1ce5316506b8c27c3e7a62482409c6153c/modules/nf-core/gatk4/samtofastq) +- [picard/samtofastq](modules/picard/samtofastq) adapts the [nf-core gatk4 samtofastq module](https://github.com/nf-core/modules/tree/ef007b1ce5316506b8c27c3e7a62482409c6153c/modules/nf-core/gatk4/samtofastq) From 42811facb3dfc83b262bf9f5c0f71c4c774afa74 Mon Sep 17 00:00:00 2001 From: Kelly Sovacool Date: Mon, 23 Oct 2023 14:20:22 -0400 Subject: [PATCH 6/6] fix: stub needs to create versions.yml --- modules/CCBR/picard/samtofastq/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/CCBR/picard/samtofastq/main.nf b/modules/CCBR/picard/samtofastq/main.nf index 7d538dc..fca6b17 100644 --- a/modules/CCBR/picard/samtofastq/main.nf +++ b/modules/CCBR/picard/samtofastq/main.nf @@ -43,6 +43,6 @@ process PICARD_SAMTOFASTQ { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${prefix}.fastq.gz + touch ${prefix}.fastq.gz versions.yml """ }