Dev -> Master for 3.11 release

.github/workflows/awsfulltest.yml

@@ -15,7 +15,7 @@ jobs:
     # Do a full-scale run with each of the three aligners
     strategy:
       matrix:
-        aligner: ["star_salmon", "star_rsem", "hisat2"]
+        aligner: ["star_salmon", "star_rsem"]
     steps:
       - name: Launch workflow via tower
         uses: nf-core/tower-action@v3

.github/workflows/ci.yml

@@ -35,9 +35,33 @@ jobs:
         with:
           version: "${{ matrix.NXF_VER }}"
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: rnaseq3_10-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: rnaseq3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Run pipeline with test data
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
 
   star_salmon:
     name: Test STAR Salmon with workflow parameters
@@ -62,14 +86,38 @@ jobs:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: rnaseq3_10-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: rnaseq3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
       - name: Run pipeline with STAR and various parameters
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
 
   star_rsem:
     name: Test STAR RSEM with workflow parameters
@@ -84,14 +132,38 @@ jobs:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: rnaseq3_10-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: rnaseq3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
       - name: Run pipeline with RSEM STAR and various parameters
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner star_rsem ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
 
   hisat2:
     name: Test HISAT2 with workflow parameters
@@ -106,14 +178,38 @@ jobs:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: rnaseq3_10-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: rnaseq3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
       - name: Run pipeline with HISAT2 and various parameters
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --aligner hisat2 ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/
 
   salmon:
     name: Test Salmon with workflow parameters
@@ -128,11 +224,35 @@ jobs:
       - name: Check out pipeline code
         uses: actions/checkout@v2
 
+      - name: Cache test data
+        id: cache-testdata
+        uses: actions/cache@v3
+        with:
+          path: test-datasets/
+          key: rnaseq3_10-test-data
+
+      - name: Check out test data
+        if: steps.cache-testdata.outputs.cache-hit != 'true'
+        uses: actions/checkout@v3
+        with:
+          repository: nf-core/test-datasets
+          ref: rnaseq3
+          path: test-datasets/
+
+      - name: Replace remote paths in samplesheets
+        run: |
+          for f in ${{ github.workspace }}/test-datasets/samplesheet/v3.10/*.csv; do
+            sed -i "s=https://raw.githubusercontent.com/nf-core/test-datasets/rnaseq/=${{ github.workspace }}/test-datasets/=g" $f
+            echo "========== $f ============"
+            cat $f
+            echo "========================================"
+          done;
+
       - name: Install Nextflow
         run: |
           wget -qO- get.nextflow.io | bash
           sudo mv nextflow /usr/local/bin/
 
       - name: Run pipeline with Salmon and various parameters
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker --pseudo_aligner salmon ${{ matrix.parameters }} --outdir ./results --test_data_base ${{ github.workspace }}/test-datasets/

CHANGELOG.md

@@ -3,6 +3,59 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [[3.11.0](https://github.com/nf-core/rnaseq/releases/tag/3.11.0)] - 2023-03-22
+
+### Credits
+
+Special thanks to the following for their code contributions to the release:
+
+- [J Lorent](https://github.com/jlorent)
+- [Luca Beltrame](https://github.com/lbeltrame)
+- [Matthias Zepper](https://github.com/MatthiasZepper)
+- [Maxime Garcia](https://github.com/maxulysse)
+- [Ryan Yordanoff](https://github.com/ryanyord)
+- [Thomas Sandmann](https://github.com/tomsing1)
+
+Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
+
+### Enhancements & fixes
+
+- [[#878](https://github.com/nf-core/rnaseq/pull/878)] - Allow tabs in fasta header when creating decoys for salmon index
+- [[#931](https://github.com/nf-core/rnaseq/pull/931)] - Save transcriptome BAM files when using `--save_umi_intermeds` / `--save_align_intermeds`
+- [[#934](https://github.com/nf-core/rnaseq/pull/934)] - Union of `ext.args` and `params.extra_star_align_args` prevents parameter clashes in the STAR module
+- [[#940](https://github.com/nf-core/rnaseq/issues/940)] - Bugfix in `salmon_summarizedexperiment.r` to ensure `rbind` doesn't fail when `rowdata` has no `tx` column. See ([[#941](https://github.com/nf-core/rnaseq/pull/941)]) for details.
+- [[#944](https://github.com/nf-core/rnaseq/issues/944)] - Read clipping using clip_r1, clip_r2, three_prime_clip_r1, three_prime_clip_r2 disabled in 3.10
+- [[#956](https://github.com/nf-core/rnaseq/pull/956)] - Implement 'auto' as default strandedness argument in `fastq_dir_to_samplesheet.py` script
+- [[#960](https://github.com/nf-core/rnaseq/pull/960)] - Failure with awsbatch when running processes that are using `executor: local`
+- [[#961](https://github.com/nf-core/rnaseq/pull/961)] - Add warnings to STDOUT for all skipped and failed strandedness check samples
+- Remove HISAT2 from automated AWS full-sized tests
+
+### Parameters
+
+| Old parameter | New parameter |
+| ------------- | ------------- |
+| `--tracedir`  |               |
+
+> **NB:** Parameter has been **updated** if both old and new parameter information is present.
+> **NB:** Parameter has been **added** if just the new parameter information is present.
+> **NB:** Parameter has been **removed** if new parameter information isn't present.
+
+### Software dependencies
+
+Note, since the pipeline is now using Nextflow DSL2, each process will be run with its own [Biocontainer](https://biocontainers.pro/#/registry). This means that on occasion it is entirely possible for the pipeline to be using different versions of the same tool. However, the overall software dependency changes compared to the last release have been listed below for reference.
+
+| Dependency  | Old version | New version |
+| ----------- | ----------- | ----------- |
+| `multiqc`   | 1.13        | 1.14        |
+| `picard`    | 2.27.4      | 3.0.0       |
+| `umi_tools` | 1.1.2       | 1.1.4       |
+
+> **NB:** Dependency has been **updated** if both old and new version information is present.
+>
+> **NB:** Dependency has been **added** if just the new version information is present.
+>
+> **NB:** Dependency has been **removed** if new version information isn't present.
+
 ## [[3.10.1](https://github.com/nf-core/rnaseq/releases/tag/3.10.1)] - 2023-01-05
 
 ### Enhancements & fixes

assets/email_template.html

@@ -34,7 +34,7 @@ <h4 style="margin-top: 0; color: inherit">nf-core/rnaseq execution completed uns
         <p>The full error message was:</p>
         <pre style="white-space: pre-wrap; overflow: visible; margin-bottom: 0">${errorReport}</pre>
       </div>
-      """ } else if(fail_percent_mapped.size() > 0) { out << """
+      """ } else if(skip_sample_count > 0) { out << """
       <div
         style="
           color: #856404;
@@ -48,12 +48,8 @@ <h4 style="margin-top: 0; color: inherit">nf-core/rnaseq execution completed uns
       >
         <h4 style="margin-top: 0; color: inherit">nf-core/rnaseq execution completed with warnings!</h4>
         <p>
-          The pipeline finished successfully, but the following samples were skipped due to very low alignment (&lt;
-          ${min_mapped_reads}%):
+          The pipeline finished successfully, but samples were skipped. Please check warnings at the top of the MultiQC report.
         </p>
-        <ul>
-          <li><code>${fail_percent_mapped.join('</code></li><li><code>')}</code></li>
-        </ul>
         <p></p>
       </div>
 

assets/email_template.txt

@@ -17,14 +17,12 @@ The full error message was:
 
 ${errorReport}
 """
-} else if (fail_percent_mapped.size() > 0) {
+} else if (skip_sample_count > 0) {
     out << """##################################################
 ## nf-core/rnaseq execution completed with warnings ##
 ##################################################
-The pipeline finished successfully, but the following samples were skipped,
-due to very low alignment (less than ${min_mapped_reads}%):
-
-  - ${fail_percent_mapped.join("\n  - ")}
+The pipeline finished successfully, but samples were skipped.
+Please check warnings at the top of the MultiQC report.
 """
 } else {
     out << "## nf-core/rnaseq execution completed successfully! ##"

bin/check_samplesheet.py

@@ -48,7 +48,6 @@ def check_samplesheet(file_in, file_out):
 
     sample_mapping_dict = {}
     with open(file_in, "r", encoding="utf-8-sig") as fin:
-
         ## Check header
         MIN_COLS = 3
         HEADER = ["sample", "fastq_1", "fastq_2", "strandedness"]
@@ -142,7 +141,6 @@ def check_samplesheet(file_in, file_out):
                 ",".join(["sample", "single_end", "fastq_1", "fastq_2", "strandedness"] + header[len(HEADER) :]) + "\n"
             )
             for sample in sorted(sample_mapping_dict.keys()):
-
                 ## Check that multiple runs of the same sample are of the same datatype i.e. single-end / paired-end
                 if not all(x[0] == sample_mapping_dict[sample][0][0] for x in sample_mapping_dict[sample]):
                     print_error(

bin/fastq_dir_to_samplesheet.py

@@ -18,8 +18,8 @@ def parse_args(args=None):
         "--strandedness",
         type=str,
         dest="STRANDEDNESS",
-        default="unstranded",
-        help="Value for 'strandedness' in samplesheet. Must be one of 'unstranded', 'forward', 'reverse'.",
+        default="auto",
+        help="Value for 'strandedness' in samplesheet. Must be one of 'unstranded', 'forward', 'reverse', 'auto'.",
     )
     parser.add_argument(
         "-r1",
@@ -80,7 +80,7 @@ def parse_args(args=None):
 def fastq_dir_to_samplesheet(
     fastq_dir,
     samplesheet_file,
-    strandedness="unstranded",
+    strandedness="auto",
     read1_extension="_R1_001.fastq.gz",
     read2_extension="_R2_001.fastq.gz",
     single_end=False,
@@ -154,8 +154,8 @@ def get_fastqs(extension, recursive=False):
 def main(args=None):
     args = parse_args(args)
 
-    strandedness = "unstranded"
-    if args.STRANDEDNESS in ["unstranded", "forward", "reverse"]:
+    strandedness = "auto"
+    if args.STRANDEDNESS in ["unstranded", "forward", "reverse", "auto"]:
         strandedness = args.STRANDEDNESS
 
     fastq_dir_to_samplesheet(

bin/filter_gtf_for_genes_in_genome.py

@@ -46,7 +46,6 @@ def extract_genes_in_genome(fasta, gtf_in, gtf_out):
     n_lines_in_genome = 0
     with open(gtf_out, "w") as f:
         with open(gtf_in) as g:
-
             for line in g.readlines():
                 n_total_lines += 1
                 seq_name_gtf = line.split("\t")[0]

bin/mqc_features_stat.py

@@ -24,7 +24,6 @@
 
 
 def mqc_feature_stat(bfile, features, outfile, sname=None):
-
     # If sample name not given use file name
     if not sname:
         sname = os.path.splitext(os.path.basename(bfile))[0]