Merge pull request #13 from EBI-Metagenomics/feature/multiqc-per-stud…

…y-and-run MultiQC reports per run set and per study
EBI-Metagenomics · Jul 11, 2024 · aafd6bf · aafd6bf
2 parents 3b57035 + a8a5131
commit aafd6bf
Show file tree

Hide file tree

Showing 19 changed files with 236 additions and 130 deletions.
diff --git a/conf/modules.config b/conf/modules.config
@@ -222,21 +222,22 @@ process {
         cpus   = { check_max( 2     * task.attempt, 'cpus'    ) }
         memory = { check_max( 12.GB * task.attempt, 'memory'  ) }
         time   = { check_max( 4.h   * task.attempt, 'time'    ) }
-
         ext.args = "--invert-match"
+    }
 
+    // Dummy process to published the filtered and decontaminated contigs
+    withName: 'PUBLISH_CLEANED_CONTIGS' {
+        cpus   = { check_max( 1                     , 'cpus'    ) }
+        memory = { check_max( 250.MB * task.attempt , 'memory'  ) }
+        time   = { check_max( 30.m    * task.attempt, 'time'    ) }
         publishDir = [
             [
                 path: "${params.outdir}",
                 mode: params.publish_dir_mode,
                 failOnError: true,
                 saveAs: {
                     filename -> {
-                        if ( filename.equals('versions.yml') ) {
-                            return null;
-                        }
-                        def output_file = new File(filename);
-                        return "${study_reads_folder( meta )}/assembly/${meta.assembler}/${meta.assembler_version}/${output_file.simpleName}_filtered_contigs.${output_file.extension}";
+                        return "${study_reads_folder( meta )}/assembly/${meta.assembler}/${meta.assembler_version}/${filename}";
                     }
                 }
             ]
@@ -288,7 +289,32 @@ process {
         ]
     }
 
-    withName: 'MULTIQC' {
+    withName: 'MULTIQC_STUDY' {
+        cpus   = { check_max( 1                  , 'cpus'    ) }
+        memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 4.h  * task.attempt, 'time'    ) }
+
+        ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
+
+        publishDir = [
+            [
+                path: "${params.outdir}",
+                mode: params.publish_dir_mode,
+                failOnError: true,
+                saveAs: {
+                    filename -> {
+                        if ( filename.equals('versions.yml') ) {
+                            return null;
+                        }
+                        def output_file = new File(filename);
+                        return "${study_folder( meta )}/multiqc/${output_file.name}";
+                    }
+                }
+            ]
+        ]
+    }
+
+    withName: 'MULTIQC_RUN' {
         cpus   = { check_max( 1                  , 'cpus'    ) }
         memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
         time   = { check_max( 4.h  * task.attempt, 'time'    ) }

diff --git a/modules.json b/modules.json
@@ -34,7 +34,7 @@
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "bba7e362e4afead70653f84d8700588ea28d0f9e",
+                        "git_sha": "82024cf6325d2ee194e7f056d841ecad2f6856e9",
                         "installed_by": ["modules"]
                     },
                     "fastp": {

diff --git a/modules/local/fetchtool_reads.nf b/modules/local/fetchtool_reads.nf
@@ -3,7 +3,7 @@ process FETCHTOOL_READS {
 
     label 'process_single'
 
-    container "microbiome-informatics/fetch-tool:v0.9.0"
+    container "quay.io/microbiome-informatics/fetch-tool:v1.0.0rc"
 
     input:
     tuple val(meta), val(study_accession), val(reads_accession)

diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml
diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf
diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py
diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test
diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap
diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf
diff --git a/nextflow.config b/nextflow.config
@@ -263,18 +263,23 @@ includeConfig 'conf/modules.config'
 
 // Function to return study / reads folder SRPXXX/SRPXXXXXX/SRRYYYY/SRRYYYYYY
 def study_reads_folder( meta = null ) {
-    def study_accession = meta?.study_accession ?: params.study_accession
     // The reads accession is used as the id for samplesheets
     def reads_accession = meta?.id ?: params.reads_accession
 
-    return [
-        study_accession.substring(0, 7),
-        study_accession,
+    return study_folder( meta ) + "/" + [
         reads_accession.substring(0, 7),
         reads_accession,
     ].join("/")
 }
 
+// Function to return study folder SRPXXX/SRPXXXXXX
+def study_folder( meta = null ) {
+    def study_accession = meta?.study_accession ?: params.study_accession
+    return [
+        study_accession.substring(0, 7),
+        study_accession,
+    ].join("/")
+}
 
 // Function to ensure that resource requirements don't go beyond
 // a maximum limit

diff --git a/subworkflows/local/assembly_qc.nf b/subworkflows/local/assembly_qc.nf
@@ -4,6 +4,20 @@ include { SEQKIT_GREP as SEQKIT_GREP_HUMAN_PHIX   } from '../../modules/nf-core/
 include { SEQKIT_GREP as SEQKIT_GREP_HOST         } from '../../modules/nf-core/seqkit/grep/main'
 include { SEQKIT_SEQ                              } from '../../modules/nf-core/seqkit/seq/main'
 
+process PUBLISH_CLEANED_CONTIGS {
+
+    input:
+    tuple val(meta), path(cleaned_contigs)
+
+    output:
+    tuple val(meta), path("${meta.id}_cleaned.contigs.fa.gz")
+
+    script:
+    """
+    cp ${cleaned_contigs} ${meta.id}_cleaned.contigs.fa.gz
+    """
+}
+
 workflow ASSEMBLY_QC {
 
     take:
@@ -69,6 +83,10 @@ workflow ASSEMBLY_QC {
         ch_versions = ch_versions.mix(SEQKIT_GREP_HOST.out.versions)
     }
 
+    PUBLISH_CLEANED_CONTIGS(
+        filtered_contigs
+    )
+
     emit:
     filtered_contigs = filtered_contigs
     versions         = ch_versions

diff --git a/tests/main.nf.test b/tests/main.nf.test
@@ -3,27 +3,28 @@ nextflow_pipeline {
     name "Test Workflow EBIMETAGENOMICS_MIASSEMBLER"
     script "main.nf"
 
-    test("Samplesheet spades") {
-
-        when {
-            params {
-                outdir = "tests/results"
-                assembler = "spades"
-                bwamem2_reference_genomes_folder = "${projectDir}/tests/human_phix/bwa2mem"
-                blast_reference_genomes_folder   = "${projectDir}/tests/human_phix/blast"
-                samplesheet                      = "${projectDir}/tests/samplesheet/test.csv"
-            }
-        }
-
-        then {
-            with(workflow) {
-                assert success
-                assert trace.succeeded().count{ task -> task.name.contains("SPADES") } == 1
-                assert trace.succeeded().count{ task -> task.name.contains("MEGAHIT") } == 1
-            }
-        }
-
-    }
+    // TODO: re-enable after the tests reads are on the repo
+    // test("Samplesheet spades") {
+
+    //     when {
+    //         params {
+    //             outdir = "tests/results"
+    //             assembler = "spades"
+    //             bwamem2_reference_genomes_folder = "${projectDir}/tests/human_phix/bwa2mem"
+    //             blast_reference_genomes_folder   = "${projectDir}/tests/human_phix/blast"
+    //             samplesheet                      = "${projectDir}/tests/samplesheet/test.csv"
+    //         }
+    //     }
+
+    //     then {
+    //         with(workflow) {
+    //             assert success
+    //             assert trace.succeeded().count{ task -> task.name.contains("SPADES") } == 1
+    //             assert trace.succeeded().count{ task -> task.name.contains("MEGAHIT") } == 1
+    //         }
+    //     }
+
+    // }
 
     test("metaSPAdes - paired end") {
 
@@ -43,7 +44,7 @@ nextflow_pipeline {
                 assert success
                 assert trace.succeeded().count{ task -> task.name.contains("SPADES") } == 1
                 assert trace.succeeded().count{ task -> task.name.contains("MEGAHIT") } == 0
-                assert trace.succeeded().size() == 16
+                assert trace.succeeded().size() == 18
             }
         }
 
@@ -67,7 +68,7 @@ nextflow_pipeline {
                 assert success
                 assert trace.succeeded().count{ task -> task.name.contains("SPADES") } == 0
                 assert trace.succeeded().count{ task -> task.name.contains("MEGAHIT") } == 1
-                assert trace.tasks().size() == 16
+                assert trace.tasks().size() == 18
             }
         }
 
@@ -133,7 +134,7 @@ nextflow_pipeline {
                 assert success
                 assert trace.succeeded().count{ task -> task.name.contains("SPADES") } == 0
                 assert trace.succeeded().count{ task -> task.name.contains("MEGAHIT") } == 1
-                assert trace.tasks().size() == 16
+                assert trace.tasks().size() == 18
             }
         }
 

diff --git a/tests/samplesheet/test.csv b/tests/samplesheet/test.csv
@@ -1,3 +1,4 @@
-study_accession,reads_accession,fastq_1,fastq_2,library_layout,library_strategy,assembler
-SRP115494,SRR6180434,ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR618/004/SRR6180434/SRR6180434_1.fastq.gz,ftp://ftp.sra.ebi.ac.uk/vol1/fastq/SRR618/004/SRR6180434/SRR6180434_2.fastq.gz,paired,metagenomic
-DRP007622,DRR280712,ftp://ftp.sra.ebi.ac.uk/vol1/fastq/DRR280/DRR280712/DRR280712.fastq.gz,,single,metatranscriptomic,megahit
+study_accession,reads_accession,fastq_1,fastq_2,library_layout,library_strategy,assembler,assembly_memory
+SRP115494,SRR6180434,tests/test_reads/SRR6180434_1.fastq.gz,tests/test_reads/SRR6180434_2.fastq.gz,paired,metagenomic,,
+SRP115494,SRR5949318,tests/test_reads/SRR5949318_1.fastq.gz,tests/test_reads/SRR5949318_2.fastq.gz,paired,metagenomic,,
+DRP007622,DRR280712,tests/test_reads/DRR280712.fastq.gz,,single,metatranscriptomic,megahit,
diff --git a/tests/test_reads/DRR280712.fastq.gz b/tests/test_reads/DRR280712.fastq.gz
diff --git a/tests/test_reads/SRR5949318_1.fastq.gz b/tests/test_reads/SRR5949318_1.fastq.gz
diff --git a/tests/test_reads/SRR5949318_2.fastq.gz b/tests/test_reads/SRR5949318_2.fastq.gz
diff --git a/tests/test_reads/SRR6180434_1.fastq.gz b/tests/test_reads/SRR6180434_1.fastq.gz
diff --git a/tests/test_reads/SRR6180434_2.fastq.gz b/tests/test_reads/SRR6180434_2.fastq.gz