Merge pull request #256 from drpatelh/template_dev

Update pipeline template with current tools dev version to fix linting

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,4 +1,3 @@
-
 name: Bug report
 description: Report something that is broken or incorrect
 labels: bug

.github/workflows/awsfulltest.yml

@@ -14,16 +14,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
+        uses: nf-core/tower-action@v3
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
-          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/chipseq/work-${{ github.sha }}
           parameters: |
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/chipseq/results-${{ github.sha }}"
             }
           profiles: test_full,aws_tower
+          nextflow_config: |
+            process.errorStrategy = 'retry'
+            process.maxRetries = 3

.github/workflows/awstest.yml

@@ -11,17 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Launch workflow via tower
-        uses: nf-core/tower-action@v2
-
+        uses: nf-core/tower-action@v3
         with:
           workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
           access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
           compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
-          pipeline: ${{ github.repository }}
-          revision: ${{ github.sha }}
           workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/chipseq/work-${{ github.sha }}
           parameters: |
             {
               "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/chipseq/results-test-${{ github.sha }}"
             }
           profiles: test,aws_tower
+          nextflow_config: |
+            process.errorStrategy = 'retry'
+            process.maxRetries = 3

.github/workflows/branch.yml

@@ -43,4 +43,3 @@ jobs:
             Thanks again for your contribution!
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allow-repeats: false
-

.github/workflows/ci.yml

@@ -53,8 +53,8 @@ jobs:
     strategy:
       matrix:
         parameters:
-        - "--skip_trimming"
-        - "--skip_consensus_peaks"
+          - "--skip_trimming"
+          - "--skip_consensus_peaks"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2
@@ -75,8 +75,8 @@ jobs:
     strategy:
       matrix:
         aligner:
-        - "bowtie2"
-        - "star"
+          - "bowtie2"
+          - "star"
     steps:
       - name: Check out pipeline code
         uses: actions/checkout@v2

.github/workflows/linting.yml

@@ -12,9 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v1
-        with:
-          node-version: '10'
+      - uses: actions/setup-node@v2
       - name: Install markdownlint
         run: npm install -g markdownlint-cli
       - name: Run Markdownlint
@@ -51,9 +49,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
-      - uses: actions/setup-node@v1
-        with:
-          node-version: '10'
+      - uses: actions/setup-node@v2
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -64,14 +60,13 @@ jobs:
   YAML:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
-      - uses: actions/setup-node@v1
+      - name: Checkout
+        uses: actions/checkout@master
+      - name: 'Yamllint'
+        uses: karancode/yamllint-github-action@master
         with:
-          node-version: '10'
-      - name: Install yaml-lint
-        run: npm install -g yaml-lint
-      - name: Run yaml-lint
-        run: yamllint $(find ${GITHUB_WORKSPACE} -type f -name "*.yml" -o -name "*.yaml")
+          yamllint_file_or_dir: '.'
+          yamllint_config_filepath: '.yamllint.yml'
 
       # If the above check failed, post a comment on the PR explaining the failure
       - name: Post PR comment
@@ -84,10 +79,11 @@ jobs:
             To keep the code consistent with lots of contributors, we run automated code consistency checks.
             To fix this CI test, please run:
 
-            * Install `yaml-lint`
-                * [Install `npm`](https://www.npmjs.com/get-npm) then [install `yaml-lint`](https://www.npmjs.com/package/yaml-lint) (`npm install -g yaml-lint`)
+            * Install `yamllint`
+                * Install `yamllint` following [this](https://yamllint.readthedocs.io/en/stable/quickstart.html#installing-yamllint)
+                instructions or alternative install it in your [conda environment](https://anaconda.org/conda-forge/yamllint)
             * Fix the markdown errors
-                * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml")`
+                * Run the test locally: `yamllint $(find . -type f -name "*.yml" -o -name "*.yaml") -c ./.yamllint.yml`
                 * Fix any reported errors in your YAML files
 
             Once you push these changes the test should pass, and you can hide this comment :+1:
@@ -142,4 +138,3 @@ jobs:
             lint_log.txt
             lint_results.md
             PR_number.txt
-

.github/workflows/linting_comment.yml

@@ -1,4 +1,3 @@
-
 name: nf-core linting comment
 # This workflow is triggered after the linting action is complete
 # It posts an automated comment to the PR, even if the PR is coming from a fork
@@ -27,4 +26,3 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           number: ${{ steps.pr_number.outputs.pr_number }}
           path: linting-logs/lint_results.md
-

.gitpod.yml

@@ -0,0 +1,14 @@
+image: nfcore/gitpod:latest
+
+vscode:
+  extensions: # based on nf-core.nf-core-extensionpack
+    - codezombiech.gitignore                 # Language support for .gitignore files
+    # - cssho.vscode-svgviewer                 # SVG viewer
+    - davidanson.vscode-markdownlint         # Markdown/CommonMark linting and style checking for Visual Studio Code
+    - eamodio.gitlens                        # Quickly glimpse into whom, why, and when a line or code block was changed
+    - EditorConfig.EditorConfig              # override user/workspace settings with settings found in .editorconfig files
+    - Gruntfuggly.todo-tree                  # Display TODO and FIXME in a tree view in the activity bar
+    - mechatroner.rainbow-csv                # Highlight columns in csv files in different colors
+    # - nextflow.nextflow                      # Nextflow syntax highlighting
+    - oderwat.indent-rainbow                 # Highlight indentation level
+    - streetsidesoftware.code-spell-checker  # Spelling checker for source code

.nf-core.yml

@@ -3,6 +3,12 @@ lint:
     - .markdownlint.yml
     - assets/email_template.html
     - assets/email_template.txt
-    - lib/NfcoreTemplate.groovy
     - assets/multiqc_config.yaml
+    - lib/NfcoreTemplate.groovy
+    - lib/NfcoreSchema.groovy
+    - .github/ISSUE_TEMPLATE/bug_report.yml
+    - .github/workflows/branch.yml
+    - .github/workflows/linting_comment.yml
+    - .github/workflows/linting.yml
 
+repository_type: pipeline

.yamllint.yml

@@ -0,0 +1,6 @@
+extends: default
+
+rules:
+    document-start: disable
+    line-length: disable
+    truthy: disable

CHANGELOG.md

@@ -33,9 +33,9 @@ via the `--aligner` parameter
 |                              | `--save_unaligned`         |
 | `--skip_diff_analysis`       | `--skip_deseq2_qc`         |
 
-> **NB:** Parameter has been __updated__ if both old and new parameter information is present.
-> **NB:** Parameter has been __added__ if just the new parameter information is present.
-> **NB:** Parameter has been __removed__ if parameter information isn't present.
+> __NB:__ Parameter has been __updated__ if both old and new parameter information is present.
+> __NB:__ Parameter has been __added__ if just the new parameter information is present.
+> __NB:__ Parameter has been __removed__ if parameter information isn't present.
 
 ### Software dependencies
 
@@ -49,9 +49,9 @@ TODO: update all new dependencies
 | `deeptools` | 3.4.3       | 3.5.1       |
 | `samtools`  | 1.10        | 1.13        |
 
-> **NB:** Dependency has been __updated__ if both old and new version information is present.
-> **NB:** Dependency has been __added__ if just the new version information is present.
-> **NB:** Dependency has been __removed__ if version information isn't present.
+> __NB:__ Dependency has been __updated__ if both old and new version information is present.
+> __NB:__ Dependency has been __added__ if just the new version information is present.
+> __NB:__ Dependency has been __removed__ if version information isn't present.
 
 ## [1.2.2] - 2021-04-22
 
@@ -156,7 +156,7 @@ TODO: update all new dependencies
 
 ### `Fixed`
 
-* **Change all parameters from `camelCase` to `snake_case` (see [Deprecated](#Deprecated))**
+* Change all parameters from `camelCase` to `snake_case` (see [Deprecated](#Deprecated))
 * [nf-core/atacseq#44](https://github.com/nf-core/atacseq/issues/44) - Output directory missing: macs2/consensus/deseq2
 * [nf-core/atacseq#45](https://github.com/nf-core/atacseq/issues/45) - Wrong x-axis scale for the HOMER: Peak annotation Counts tab plot?
 * [nf-core/atacseq#46](https://github.com/nf-core/atacseq/issues/46) - Stage blacklist file in channel properly

README.md

@@ -37,10 +37,10 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool
         * reads that are unmapped ([`SAMtools`](https://sourceforge.net/projects/samtools/files/samtools/))
         * reads that map to multiple locations ([`SAMtools`](https://sourceforge.net/projects/samtools/files/samtools/))
         * reads containing > 4 mismatches ([`BAMTools`](https://github.com/pezmaster31/bamtools))
-        * reads that have an insert size > 2kb ([`BAMTools`](https://github.com/pezmaster31/bamtools); *paired-end only*)
-        * reads that map to different chromosomes ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); *paired-end only*)
-        * reads that arent in FR orientation ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); *paired-end only*)
-        * reads where only one read of the pair fails the above criteria ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); *paired-end only*)
+        * reads that have an insert size > 2kb ([`BAMTools`](https://github.com/pezmaster31/bamtools); _paired-end only_)
+        * reads that map to different chromosomes ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); _paired-end only_)
+        * reads that arent in FR orientation ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); _paired-end only_)
+        * reads where only one read of the pair fails the above criteria ([`Pysam`](http://pysam.readthedocs.io/en/latest/installation.html); _paired-end only_)
     3. Alignment-level QC and estimation of library complexity ([`picard`](https://broadinstitute.github.io/picard/), [`Preseq`](http://smithlabresearch.org/software/preseq/))
     4. Create normalised bigWig files scaled to 1 million mapped reads ([`BEDTools`](https://github.com/arq5x/bedtools2/), [`bedGraphToBigWig`](http://hgdownload.soe.ucsc.edu/admin/exe/))
     5. Generate gene-body meta-profile from bigWig files ([`deepTools`](https://deeptools.readthedocs.io/en/develop/content/tools/plotProfile.html))

assets/multiqc_config.yaml

@@ -21,56 +21,56 @@ exclude_modules:
 
 module_order:
     - fastqc:
-        name: 'LIB: FastQC (raw)'
-        info: 'This section of the report shows FastQC results before adapter trimming for individual libraries.'
-        path_filters:
-            - './fastqc/*.zip'
+          name: 'LIB: FastQC (raw)'
+          info: 'This section of the report shows FastQC results before adapter trimming for individual libraries.'
+          path_filters:
+              - './fastqc/*.zip'
     - cutadapt:
-        name: 'LIB: cutadapt (trimmed)'
-        info: 'This section of the report shows the length of trimmed reads by cutadapt for individual libraries.'
+          name: 'LIB: cutadapt (trimmed)'
+          info: 'This section of the report shows the length of trimmed reads by cutadapt for individual libraries.'
     - fastqc:
-        name: 'LIB: FastQC (trimmed)'
-        info: 'This section of the report shows FastQC results after adapter trimming for individual libraries.'
-        path_filters:
-            - './trimgalore/fastqc/*.zip'
+          name: 'LIB: FastQC (trimmed)'
+          info: 'This section of the report shows FastQC results after adapter trimming for individual libraries.'
+          path_filters:
+              - './trimgalore/fastqc/*.zip'
     - samtools:
-        name: 'LIB: SAMTools'
-        info: 'This section of the report shows SAMTools results for individual libraries.'
-        path_filters:
-            - './alignment/library/*'
+          name: 'LIB: SAMTools'
+          info: 'This section of the report shows SAMTools results for individual libraries.'
+          path_filters:
+              - './alignment/library/*'
     - samtools:
-        name: 'MERGED LIB: SAMTools (unfiltered)'
-        info: 'This section of the report shows SAMTools results after merging libraries and before filtering.'
-        path_filters:
-            - './alignment/mergedLibrary/unfiltered/*.mLb.mkD.sorted.bam*'
+          name: 'MERGED LIB: SAMTools (unfiltered)'
+          info: 'This section of the report shows SAMTools results after merging libraries and before filtering.'
+          path_filters:
+              - './alignment/mergedLibrary/unfiltered/*.mLb.mkD.sorted.bam*'
     - picard:
-        name: 'MERGED LIB: Picard (unfiltered)'
-        info: 'This section of the report shows picard results after merging libraries and before filtering.'
-        path_filters:
-            - './alignment/mergedLibrary/unfiltered/picard_metrics/*'
+          name: 'MERGED LIB: Picard (unfiltered)'
+          info: 'This section of the report shows picard results after merging libraries and before filtering.'
+          path_filters:
+              - './alignment/mergedLibrary/unfiltered/picard_metrics/*'
     - preseq:
-        name: 'MERGED LIB: Preseq (unfiltered)'
-        info: 'This section of the report shows Preseq results after merging libraries and before filtering.'
+          name: 'MERGED LIB: Preseq (unfiltered)'
+          info: 'This section of the report shows Preseq results after merging libraries and before filtering.'
     - samtools:
-        name: 'MERGED LIB: SAMTools (filtered)'
-        info: 'This section of the report shows SAMTools results after merging libraries and after filtering.'
-        path_filters:
-            - './alignment/mergedLibrary/filtered/*.mLb.clN.sorted.bam*'
+          name: 'MERGED LIB: SAMTools (filtered)'
+          info: 'This section of the report shows SAMTools results after merging libraries and after filtering.'
+          path_filters:
+              - './alignment/mergedLibrary/filtered/*.mLb.clN.sorted.bam*'
     - picard:
-        name: 'MERGED LIB: Picard (filtered)'
-        info: 'This section of the report shows picard results after merging libraries and after filtering.'
-        path_filters:
-            - './alignment/mergedLibrary/filtered/picard_metrics/*'
+          name: 'MERGED LIB: Picard (filtered)'
+          info: 'This section of the report shows picard results after merging libraries and after filtering.'
+          path_filters:
+              - './alignment/mergedLibrary/filtered/picard_metrics/*'
     - deeptools:
-        name: 'MERGED LIB: deepTools'
-        anchor: 'mlib_deeptools'
-        info: 'This section of the report shows ChIP-seq QC plots generated by deepTools.'
+          name: 'MERGED LIB: deepTools'
+          anchor: 'mlib_deeptools'
+          info: 'This section of the report shows ChIP-seq QC plots generated by deepTools.'
     - featureCounts:
-        name: 'MERGED LIB: featureCounts'
-        anchor: 'mlib_featurecounts'
-        info: 'This section of the report shows featureCounts results for the number of reads assigned to merged library consensus peaks.'
-        path_filters:
-            - './macs/consensus/*.summary'
+          name: 'MERGED LIB: featureCounts'
+          anchor: 'mlib_featurecounts'
+          info: 'This section of the report shows featureCounts results for the number of reads assigned to merged library consensus peaks.'
+          path_filters:
+              - './macs/consensus/*.summary'
 
 report_section_order:
     peak_count:

conf/base.config

@@ -1,7 +1,7 @@
 /*
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     nf-core/chipseq Nextflow base config file
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     A 'blank slate' config file, appropriate for general use on most high performance
     compute environments. Assumes that all software is installed and available on
     the PATH. Runs in `local` mode - all jobs will be run on the logged in environment.

conf/igenomes.config

@@ -1,7 +1,7 @@
 /*
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Nextflow config file for iGenomes paths
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Defines reference genomes using iGenome paths.
     Can be used by any config that customises the base path using:
         $params.igenomes_base / --igenomes_base

conf/modules.config

@@ -1,7 +1,7 @@
 /*
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Config file for defining DSL2 per module options and publishing paths
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Available keys to override module options:
         ext.args            = Additional arguments appended to command in module.
         ext.args2           = Second set of arguments appended to command in module (multi-tool modules).
@@ -17,22 +17,22 @@
 process {
     publishDir = [
         path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
-        mode: 'copy',
+        mode: params.publish_dir_mode,
         saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
     ]
 
     withName: 'NFCORE_CHIPSEQ:CHIPSEQ:INPUT_CHECK:SAMPLESHEET_CHECK' {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
-            mode: 'copy',
+            mode: params.publish_dir_mode,
             saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
         ]
     }
 
     withName: CUSTOM_DUMPSOFTWAREVERSIONS {
         publishDir = [
             path: { "${params.outdir}/pipeline_info" },
-            mode: 'copy',
+            mode: params.publish_dir_mode,
             pattern: '*_versions.yml'
         ]
     }

conf/test.config

@@ -1,7 +1,7 @@
 /*
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Nextflow config file for running minimal tests
-========================================================================================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Defines input files and everything required to run a fast and simple pipeline test.
 
     Use as follows: