Merge branch 'dev' into muffato-patch-1

nf-core · May 15, 2023 · bd10e58 · bd10e58
2 parents 6a95bf3 + 2dc8d24
commit bd10e58
Show file tree

Hide file tree

Showing 15 changed files with 120 additions and 49 deletions.
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -1,10 +1,11 @@
-image: nfcore/gitpod:latest
+image: nfcore/gitpod:dev
 tasks:
   - name: install current state of nf-core/tools and setup pre-commit
     command: |
       python -m pip install -e .
       python -m pip install -r requirements-dev.txt
       pre-commit install --install-hooks
+      nextflow self-update
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,11 +1,32 @@
 # nf-core/tools: Changelog
 
-# v2.9 (TBC)
+# v2.9dev
 
 ### Template
 
+- `params.max_multiqc_email_size` is no longer required
+- Remove `cleanup = true` from `test_full.config` in pipeline template
+- Fix usage docs for specifying `params.yaml`
+- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk]
+- Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286)
+- Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287)
 - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291))
 
+### Linting
+
+- Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270))
+- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])(https://github.com/nf-core/tools/pull/2278/files)
+- Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281))
+
+### Modules
+
+### Subworkflows
+
+### General
+
+- GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency.
+- GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config.
+
 # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27]
 
 ### Template

diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile
@@ -38,13 +38,17 @@ RUN conda config --add channels defaults && \
     conda config --set channel_priority strict && \
     conda install --quiet --yes --name base mamba && \
     mamba install --quiet --yes --name base \
-    nextflow=22.10.1 \
-    nf-core \
-    nf-test \
-    black \
-    prettier \
-    pytest-workflow && \
+        nextflow \
+        nf-core \
+        nf-test \
+        black \
+        prettier \
+        pre-commit \
+        pytest-workflow && \
     mamba clean --all -f -y
 
+# Update Nextflow
+RUN nextflow self-update
+
 # Install nf-core
 RUN python -m pip install .
diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf
@@ -94,4 +94,26 @@ process {{ component_name_underscore|upper }} {
         {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' ))
     END_VERSIONS
     """
+
+    stub:
+    def args = task.ext.args ?: ''
+    {% if has_meta -%}
+    def prefix = task.ext.prefix ?: "${meta.id}"
+    {%- endif %}
+    {% if not_empty_template -%}
+    // TODO nf-core: A stub section should mimic the execution of the original module as best as possible
+    //               Have a look at the following examples:
+    //               Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63
+    //               Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54
+    {%- endif %}
+    """
+    {% if not_empty_template -%}
+    touch ${prefix}.bam
+    {%- endif %}
+
+    cat <<-END_VERSIONS > versions.yml
+    "${task.process}":
+        {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' ))
+    END_VERSIONS
+    """
 }
diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py
@@ -271,15 +271,7 @@ def check_process_section(self, lines, fix_version, progress_bar):
                 self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf))
                 singularity_tag = None
             url = urlparse(l.split("'")[0])
-            # lint double quotes
-            if l.count('"') > 2:
-                self.failed.append(
-                    (
-                        "container_links",
-                        "Too many double quotes found when specifying singularity container",
-                        self.main_nf,
-                    )
-                )
+
         if _container_type(l) == "docker":
             # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5
             # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1
@@ -289,22 +281,44 @@ def check_process_section(self, lines, fix_version, progress_bar):
                 self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf))
             else:
                 self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf))
-                docker_tag = None
-            if l.startswith("biocontainers/"):
-                # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix
-                l = "quay.io/" + l
-            url = urlparse(l.split("'")[0])
-            # lint double quotes
-            if l.count('"') > 2:
+                docker_tag = NoneD
+            if l.startswith("quay.io/"):
+                l_stripped = re.sub("\W+$", "", l)
                 self.failed.append(
-                    ("container_links", "Too many double quotes found when specifying docker container", self.main_nf)
+                    (
+                        "container_links",
+                        f"{l_stripped} container name found, please use just 'organisation/container:tag' instead.",
+                        self.main_nf,
+                    )
                 )
+            else:
+                self.passed.append(("container_links", f"Container prefix is correct", self.main_nf))
+
+            # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04)
+            # If so, add quay.io as default container prefix
+            if l.count("/") == 1 and l.count(":") == 1:
+                l = "quay.io/" + l
+            url = urlparse(l.split("'")[0])
+
         # lint double quotes
-        if l.startswith("container"):
+        if l.startswith("container") or _container_type(l) == "docker" or _container_type(l) == "singularity":
             if l.count('"') > 2:
                 self.failed.append(
-                    ("container_links", "Too many double quotes found when specifying containers", self.main_nf)
+                    (
+                        "container_links",
+                        f"Too many double quotes found when specifying container: {l.lstrip('container ')}",
+                        self.main_nf,
+                    )
                 )
+            else:
+                self.passed.append(
+                    (
+                        "container_links",
+                        f"Correct number of double quotes found when specifying container: {l.lstrip('container ')}",
+                        self.main_nf,
+                    )
+                )
+
         # lint more than one container in the same line
         if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l):
             self.warned.append(
@@ -331,8 +345,14 @@ def check_process_section(self, lines, fix_version, progress_bar):
             log.debug(f"Unable to connect to url '{urlunparse(url)}' due to error: {e}")
             self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf))
             continue
-        if response.status_code != 200:
-            self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf))
+        if not response.ok:
+            self.failed.append(
+                (
+                    "container_links",
+                    f"Unable to connect to {response.url}, status code: {response.status_code}",
+                    self.main_nf,
+                )
+            )
 
     # Check that all bioconda packages have build numbers
     # Also check for newer versions
@@ -581,9 +601,5 @@ def _container_type(line):
         if url_match:
             return "singularity"
         return None
-    if (
-        line.startswith("biocontainers/")
-        or line.startswith("quay.io/")
-        or (line.count("/") == 1 and line.count(":") == 1)
-    ):
+    if line.count("/") >= 1 and line.count(":") == 1:
         return "docker"
diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml
@@ -27,7 +27,7 @@ jobs:
             {
               "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}"
             }
-          profiles: test_full,aws_tower
+          profiles: test_full
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file

diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml
@@ -22,7 +22,7 @@ jobs:
             {
               "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}"
             }
-          profiles: test,aws_tower
+          profiles: test
       - uses: actions/upload-artifact@v3
         with:
           name: Tower debug log file

diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml
@@ -1,4 +1,9 @@
 image: nfcore/gitpod:latest
+tasks:
+  - name: Update Nextflow and setup pre-commit
+    command: |
+      pre-commit install --install-hooks
+      nextflow self-update
 
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack

diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md
@@ -82,11 +82,11 @@ nextflow run {{ name }} \
 
 {% if branded -%}
 
-For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters).
+For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters).
 
 ## Pipeline output
 
-To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page.
+To see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page.
 For more details about the output files and reports, please refer to the
 [output documentation](https://nf-co.re/{{ short_name }}/output).
 

diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config
@@ -10,8 +10,6 @@
 ----------------------------------------------------------------------------------------
 */
 
-cleanup = true
-
 params {
     config_profile_name        = 'Full test profile'
     config_profile_description = 'Full test dataset to check pipeline function'

diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md
@@ -61,7 +61,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p
 The typical command for running the pipeline is as follows:
 
 ```bash
-nextflow run {{ name }} --input samplesheet.csv --outdir <OUTDIR> --genome GRCh37 -profile docker
+nextflow run {{ name }} --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker
 ```
 
 This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles.
@@ -80,7 +80,8 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than
 Pipeline settings can be provided in a `yaml` or `json` file via `-params-file <file>`.
 
 > ⚠️ Do not use `-c <file>` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args).
-> The above pipeline run specified with a params file in yaml format:
+
+The above pipeline run specified with a params file in yaml format:
 
 ```bash
 nextflow run {{ name }} -profile docker -params-file params.yaml
@@ -92,7 +93,6 @@ with `params.yaml` containing:
 input: './samplesheet.csv'
 outdir: './results/'
 genome: 'GRCh37'
-input: 'data'
 <...>
 ```
 

diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy
@@ -128,7 +128,7 @@ class NfcoreTemplate {
         def email_html    = html_template.toString()
 
         // Render the sendmail template
-        def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit
+        def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit
         def smail_fields           = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ]
         def sf                     = new File("$projectDir/assets/sendmail_template.txt")
         def sendmail_template      = engine.createTemplate(sf).make(smail_fields)

diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config
@@ -84,7 +84,7 @@ profiles {
     debug {
         dumpHashes             = true
         process.beforeScript   = 'echo $HOSTNAME'
-        cleanup = false
+        cleanup                = false
     }
     conda {
         conda.enabled          = true
@@ -107,7 +107,6 @@ profiles {
     }
     docker {
         docker.enabled         = true
-        docker.registry        = 'quay.io'
         docker.userEmulation   = true
         conda.enabled          = false
         singularity.enabled    = false
@@ -131,7 +130,6 @@ profiles {
     }
     podman {
         podman.enabled         = true
-        podman.registry        = 'quay.io'
         conda.enabled          = false
         docker.enabled         = false
         singularity.enabled    = false
@@ -175,6 +173,12 @@ profiles {
     test_full { includeConfig 'conf/test_full.config' }
 }
 
+// Set default registry for Docker and Podman independent of -profile
+// Will not be used unless Docker / Podman are enabled
+// Set to your registry if you have a mirror of containers
+docker.registry = 'quay.io'
+podman.registry = 'quay.io'
+
 {% if igenomes %}
 // Load igenomes.config if required
 if (!params.igenomes_ignore) {

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-version = "2.8"
+version = "2.9dev"
 
 with open("README.md") as f:
     readme = f.read()

diff --git a/tests/modules/patch.py b/tests/modules/patch.py
@@ -18,7 +18,7 @@
 """
 
 ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2"
-CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8"
+CORRECT_SHA = "1dff30bfca2d98eb7ac7b09269a15e822451d99f"
 SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e"
 FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73"
 BISMARK_ALIGN = "bismark/align"