Better publishing for all aligner indexes (#53)

* update CHANGELOG * Better publishing for all aligner indexes * fix kallisto * update snapshots
nf-core · Dec 5, 2024 · 4db5898 · 4db5898
1 parent 46408bc
commit 4db5898
Show file tree

Hide file tree

Showing 10 changed files with 201 additions and 182 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -43,6 +43,7 @@ Initial release of nf-core/references, created with the [nf-core](https://nf-co.
 - [48](https://github.com/nf-core/references/pull/48) - Simplify VCF tabix index generation and related assets
 - [48](https://github.com/nf-core/references/pull/48) - Code refactoring (new subworfklows for each type of operations)
 - [49](https://github.com/nf-core/references/pull/49) - Better publishing for all files
+- [53](https://github.com/nf-core/references/pull/53) - Better publishing for all aligner indexes
 
 ### Fixed
 

diff --git a/main.nf b/main.nf
@@ -103,19 +103,19 @@ workflow {
 
 output {
     'bowtie1_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BowtieIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BowtieIndex/version1.3.1" } }
     }
     'bowtie2_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/Bowtie2Index/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/Bowtie2Index/version2.5.2" } }
     }
     'bwamem1_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BWAIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BWAIndex/version0.7.18" } }
     }
     'bwamem2_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BWAmem2Index/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/BWAmem2Index/version2.2.1" } }
     }
     'dragmap_hashmap' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/dragmap/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/dragmap/version1.2.1" } }
     }
     'fasta' {
         path { meta, fasta -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/WholeGenomeFasta/${file}" } }
@@ -133,13 +133,13 @@ output {
         path { meta, intervals -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/Genes/${file}" } }
     }
     'hisat2_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/Hisat2Index/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/Hisat2Index/version2.2.1" } }
     }
     'intervals_bed' {
         path { meta, intervals -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/intervals/${file}" } }
     }
     'kallisto_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/KallistoIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/KallistoIndex/version0.51.1/${file}" } }
     }
     'msisensorpro_list' {
         path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Annotation/msisensorpro/${file}" } }
@@ -154,16 +154,16 @@ output {
         path { folder -> { file -> "multiqc/multiqc_report" } }
     }
     'rsem_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/RSEMIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/RSEMIndex/version1.3.1" } }
     }
     'salmon_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/SalmonIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/SalmonIndex/version1.10.3" } }
     }
     'splice_sites' {
         path { meta, txt -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/SpliceSites/${file}" } }
     }
     'star_index' {
-        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/STARIndex/" } }
+        path { meta, index -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/STARIndex/version2.7.11b" } }
     }
     'transcript_fasta' {
         path { meta, fasta -> { file -> "${meta.species}/${meta.source}/${meta.id}/Sequence/TranscriptFasta/${file}" } }

diff --git a/tests/.nftignore b/tests/.nftignore
@@ -1,14 +1,14 @@
-**/KallistoIndex
-**/RSEMIndex/Log.out
-**/STARIndex/Log.out
-**/SalmonIndex/ctable.bin
-**/SalmonIndex/pos.bin
-**/SalmonIndex/pre_indexing.log
-**/SalmonIndex/ref_indexing.log
-**/SalmonIndex/seq.bin
-**/dragmap/hash_table.cfg
-**/dragmap/hash_table.cfg.bin
-**/dragmap/hash_table_stats.txt
+**/kallisto
+**/RSEMIndex/*/Log.out
+**/STARIndex/*/Log.out
+**/SalmonIndex/*/ctable.bin
+**/SalmonIndex/*/pos.bin
+**/SalmonIndex/*/pre_indexing.log
+**/SalmonIndex/*/ref_indexing.log
+**/SalmonIndex/*/seq.bin
+**/dragmap/*/hash_table.cfg
+**/dragmap/*/hash_table.cfg.bin
+**/dragmap/*/hash_table_stats.txt
 .DS_Store
 multiqc/multiqc_data/multiqc.log
 multiqc/multiqc_data/multiqc_data.json

diff --git a/tests/hisat2.nf.test.snap b/tests/hisat2.nf.test.snap
@@ -17,14 +17,15 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.1.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.2.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.3.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.4.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.5.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.6.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.7.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.8.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.1.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.2.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.3.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.4.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.5.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.6.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.7.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.8.ht2",
                 "multiqc",
                 "multiqc/multiqc_data",
                 "multiqc/multiqc_data/multiqc.log",
@@ -52,7 +53,7 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-03T12:55:26.231501216"
+        "timestamp": "2024-12-05T15:35:30.36657085"
     },
     "Run with profile test for hisat2 | --input GRCh38_chr21.yml": {
         "content": [
@@ -81,14 +82,15 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Annotation/Genes/GRCh38_chr21.gtf",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.1.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.2.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.3.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.4.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.5.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.6.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.7.ht2",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/GRCh38_chr21.8.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.1.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.2.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.3.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.4.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.5.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.6.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.7.ht2",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/Hisat2Index/version2.2.1/GRCh38_chr21.8.ht2",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/SpliceSites",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/SpliceSites/GRCh38_chr21.splice_sites.txt",
                 "multiqc",
@@ -120,6 +122,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-03T13:57:17.25040689"
+        "timestamp": "2024-12-05T15:36:59.914837757"
     }
 }
diff --git a/tests/kallisto.nf.test.snap b/tests/kallisto.nf.test.snap
@@ -27,6 +27,8 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Annotation/Genes/GRCh38_chr21.gtf",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1/kallisto",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/TranscriptFasta",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/TranscriptFasta/genome.transcripts.fa",
                 "multiqc",
@@ -50,7 +52,7 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-03T14:31:39.422746281"
+        "timestamp": "2024-12-05T15:55:35.924356749"
     },
     "Run with profile test for kallisto | --input assets/test/default_extended.yml": {
         "content": [
@@ -70,6 +72,8 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/KallistoIndex/version0.51.1/kallisto",
                 "multiqc",
                 "multiqc/multiqc_data",
                 "multiqc/multiqc_data/multiqc.log",
@@ -89,6 +93,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-03T14:31:07.501838885"
+        "timestamp": "2024-12-05T15:55:10.518283725"
     }
 }
diff --git a/tests/multiple.nf.test.snap b/tests/multiple.nf.test.snap
@@ -78,11 +78,12 @@
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Annotation/intervals/GRCh38_chr21.bed",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/GRCh38_chr21.amb",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/GRCh38_chr21.ann",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/GRCh38_chr21.bwt",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/GRCh38_chr21.pac",
-                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/GRCh38_chr21.sa",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18/GRCh38_chr21.amb",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18/GRCh38_chr21.ann",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18/GRCh38_chr21.bwt",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18/GRCh38_chr21.pac",
+                "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/BWAIndex/version0.7.18/GRCh38_chr21.sa",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/WholeGenomeFasta",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/WholeGenomeFasta/GRCh38_chr21.dict",
                 "Homo_sapiens/nf-core/references/GRCh38_chr21/Sequence/WholeGenomeFasta/GRCh38_chr21.fa.fai",
@@ -92,11 +93,12 @@
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Annotation/intervals/testdata.GRCh38_chr22.bed",
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence",
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex",
-                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/genome.amb",
-                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/genome.ann",
-                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/genome.bwt",
-                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/genome.pac",
-                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/genome.sa",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18/genome.amb",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18/genome.ann",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18/genome.bwt",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18/genome.pac",
+                "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/BWAIndex/version0.7.18/genome.sa",
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/WholeGenomeFasta",
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/WholeGenomeFasta/genome.dict",
                 "Homo_sapiens/nf-core/references/testdata.GRCh38_chr22/Sequence/WholeGenomeFasta/genome.fasta.fai",
@@ -135,6 +137,6 @@
             "nf-test": "0.9.2",
             "nextflow": "24.10.2"
         },
-        "timestamp": "2024-12-02T12:27:31.034085088"
+        "timestamp": "2024-12-05T15:38:53.100305759"
     }
 }