diff --git a/.gitpod.yml b/.gitpod.yml index 263fcc41d..1cc63b197 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -1,10 +1,11 @@ -image: nfcore/gitpod:latest +image: nfcore/gitpod:dev tasks: - name: install current state of nf-core/tools and setup pre-commit command: | python -m pip install -e . python -m pip install -r requirements-dev.txt pre-commit install --install-hooks + nextflow self-update vscode: extensions: # based on nf-core.nf-core-extensionpack - codezombiech.gitignore # Language support for .gitignore files diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b8d681ca..b79fd410d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,32 @@ # nf-core/tools: Changelog -# v2.9 (TBC) +# v2.9dev ### Template +- `params.max_multiqc_email_size` is no longer required +- Remove `cleanup = true` from `test_full.config` in pipeline template +- Fix usage docs for specifying `params.yaml` +- Added stub in modules template ([#2277])(https://github.com/nf-core/tools/pull/2277) [Contributed by @nvnieuwk] +- Move registry definitions out of profile scope ([#2286])(https://github.com/nf-core/tools/pull/2286) +- Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) +### Linting + +- Warn if container access is denied ([#2270](https://github.com/nf-core/tools/pull/2270)) +- Error if module container specification has quay.io as prefix when it shouldn't have ([#2278])(https://github.com/nf-core/tools/pull/2278/files) +- Detect if container is 'simple name' and try to contact quay.io server by default ([#2281](https://github.com/nf-core/tools/pull/2281)) + +### Modules + +### Subworkflows + +### General + +- GitPod base image: Always self-update to the latest version of Nextflow. Add [pre-commit](https://pre-commit.com/) dependency. +- GitPod configs: Update Nextflow as an init task, init pre-commit in pipeline config. + # [v2.8 - Ruthenium Monkey](https://github.com/nf-core/tools/releases/tag/2.8) - [2023-04-27] ### Template diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 417208a20..5fce2f055 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -38,13 +38,17 @@ RUN conda config --add channels defaults && \ conda config --set channel_priority strict && \ conda install --quiet --yes --name base mamba && \ mamba install --quiet --yes --name base \ - nextflow=22.10.1 \ - nf-core \ - nf-test \ - black \ - prettier \ - pytest-workflow && \ + nextflow \ + nf-core \ + nf-test \ + black \ + prettier \ + pre-commit \ + pytest-workflow && \ mamba clean --all -f -y +# Update Nextflow +RUN nextflow self-update + # Install nf-core RUN python -m pip install . diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 83cdf90b9..404d38094 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -94,4 +94,26 @@ process {{ component_name_underscore|upper }} { {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) END_VERSIONS """ + + stub: + def args = task.ext.args ?: '' + {% if has_meta -%} + def prefix = task.ext.prefix ?: "${meta.id}" + {%- endif %} + {% if not_empty_template -%} + // TODO nf-core: A stub section should mimic the execution of the original module as best as possible + // Have a look at the following examples: + // Simple example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bcftools/annotate/main.nf#L47-L63 + // Complex example: https://github.com/nf-core/modules/blob/818474a292b4860ae8ff88e149fbcda68814114d/modules/nf-core/bedtools/split/main.nf#L38-L54 + {%- endif %} + """ + {% if not_empty_template -%} + touch ${prefix}.bam + {%- endif %} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) + END_VERSIONS + """ } diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bacc7d7fb..8150e7e83 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -271,15 +271,7 @@ def check_process_section(self, lines, fix_version, progress_bar): self.failed.append(("singularity_tag", "Unable to parse singularity tag", self.main_nf)) singularity_tag = None url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: - self.failed.append( - ( - "container_links", - "Too many double quotes found when specifying singularity container", - self.main_nf, - ) - ) + if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 @@ -289,22 +281,44 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) - docker_tag = None - if l.startswith("biocontainers/"): - # When we think it is a biocontainer, assume we are querying quay.io/biocontainers and insert quay.io as prefix - l = "quay.io/" + l - url = urlparse(l.split("'")[0]) - # lint double quotes - if l.count('"') > 2: + docker_tag = NoneD + if l.startswith("quay.io/"): + l_stripped = re.sub("\W+$", "", l) self.failed.append( - ("container_links", "Too many double quotes found when specifying docker container", self.main_nf) + ( + "container_links", + f"{l_stripped} container name found, please use just 'organisation/container:tag' instead.", + self.main_nf, + ) ) + else: + self.passed.append(("container_links", f"Container prefix is correct", self.main_nf)) + + # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) + # If so, add quay.io as default container prefix + if l.count("/") == 1 and l.count(":") == 1: + l = "quay.io/" + l + url = urlparse(l.split("'")[0]) + # lint double quotes - if l.startswith("container"): + if l.startswith("container") or _container_type(l) == "docker" or _container_type(l) == "singularity": if l.count('"') > 2: self.failed.append( - ("container_links", "Too many double quotes found when specifying containers", self.main_nf) + ( + "container_links", + f"Too many double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) ) + else: + self.passed.append( + ( + "container_links", + f"Correct number of double quotes found when specifying container: {l.lstrip('container ')}", + self.main_nf, + ) + ) + # lint more than one container in the same line if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): self.warned.append( @@ -331,8 +345,14 @@ def check_process_section(self, lines, fix_version, progress_bar): log.debug(f"Unable to connect to url '{urlunparse(url)}' due to error: {e}") self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) continue - if response.status_code != 200: - self.failed.append(("container_links", "Unable to connect to container URL", self.main_nf)) + if not response.ok: + self.failed.append( + ( + "container_links", + f"Unable to connect to {response.url}, status code: {response.status_code}", + self.main_nf, + ) + ) # Check that all bioconda packages have build numbers # Also check for newer versions @@ -581,9 +601,5 @@ def _container_type(line): if url_match: return "singularity" return None - if ( - line.startswith("biocontainers/") - or line.startswith("quay.io/") - or (line.count("/") == 1 and line.count(":") == 1) - ): + if line.count("/") >= 1 and line.count(":") == 1: return "docker" diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4942167b1..9511ea57b 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -27,7 +27,7 @@ jobs: { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } - profiles: test_full,aws_tower + profiles: test_full - uses: actions/upload-artifact@v3 with: name: Tower debug log file diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 7f80cf1bb..6e1fd8c92 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -22,7 +22,7 @@ jobs: { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } - profiles: test,aws_tower + profiles: test - uses: actions/upload-artifact@v3 with: name: Tower debug log file diff --git a/nf_core/pipeline-template/.gitpod.yml b/nf_core/pipeline-template/.gitpod.yml index 85d95ecc8..25488dcc0 100644 --- a/nf_core/pipeline-template/.gitpod.yml +++ b/nf_core/pipeline-template/.gitpod.yml @@ -1,4 +1,9 @@ image: nfcore/gitpod:latest +tasks: + - name: Update Nextflow and setup pre-commit + command: | + pre-commit install --install-hooks + nextflow self-update vscode: extensions: # based on nf-core.nf-core-extensionpack diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index e2ca15a8e..5ac65123e 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -82,11 +82,11 @@ nextflow run {{ name }} \ {% if branded -%} -For more details, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). +For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/{{ short_name }}/usage) and the [parameter documentation](https://nf-co.re/{{ short_name }}/parameters). ## Pipeline output -To see the the results of a test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page. +To see the results of an example test run with a full size dataset refer to the [results](https://nf-co.re/{{ short_name }}/results) tab on the nf-core website pipeline page. For more details about the output files and reports, please refer to the [output documentation](https://nf-co.re/{{ short_name }}/output). diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index 46b165a91..d92692fa9 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -10,8 +10,6 @@ ---------------------------------------------------------------------------------------- */ -cleanup = true - params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index 73e113254..e89c2a733 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -61,7 +61,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```bash -nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile docker +nextflow run {{ name }} --input ./samplesheet.csv --outdir ./results --genome GRCh37 -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. @@ -80,7 +80,8 @@ If you wish to repeatedly use the same parameters for multiple runs, rather than Pipeline settings can be provided in a `yaml` or `json` file via `-params-file `. > ⚠️ Do not use `-c ` to specify parameters as this will result in errors. Custom config files specified with `-c` must only be used for [tuning process resource specifications](https://nf-co.re/docs/usage/configuration#tuning-workflow-resources), other infrastructural tweaks (such as output directories), or module arguments (args). -> The above pipeline run specified with a params file in yaml format: + +The above pipeline run specified with a params file in yaml format: ```bash nextflow run {{ name }} -profile docker -params-file params.yaml @@ -92,7 +93,6 @@ with `params.yaml` containing: input: './samplesheet.csv' outdir: './results/' genome: 'GRCh37' -input: 'data' <...> ``` diff --git a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy index 2cb8b4138..a1a726d69 100755 --- a/nf_core/pipeline-template/lib/NfcoreTemplate.groovy +++ b/nf_core/pipeline-template/lib/NfcoreTemplate.groovy @@ -128,7 +128,7 @@ class NfcoreTemplate { def email_html = html_template.toString() // Render the sendmail template - def max_multiqc_email_size = params.max_multiqc_email_size as nextflow.util.MemoryUnit + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("$projectDir/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 4ef0fcd5e..2650572c9 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -84,7 +84,7 @@ profiles { debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' - cleanup = false + cleanup = false } conda { conda.enabled = true @@ -107,7 +107,6 @@ profiles { } docker { docker.enabled = true - docker.registry = 'quay.io' docker.userEmulation = true conda.enabled = false singularity.enabled = false @@ -131,7 +130,6 @@ profiles { } podman { podman.enabled = true - podman.registry = 'quay.io' conda.enabled = false docker.enabled = false singularity.enabled = false @@ -175,6 +173,12 @@ profiles { test_full { includeConfig 'conf/test_full.config' } } +// Set default registry for Docker and Podman independent of -profile +// Will not be used unless Docker / Podman are enabled +// Set to your registry if you have a mirror of containers +docker.registry = 'quay.io' +podman.registry = 'quay.io' + {% if igenomes %} // Load igenomes.config if required if (!params.igenomes_ignore) { diff --git a/setup.py b/setup.py index 9b0d9fb6a..443885072 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -version = "2.8" +version = "2.9dev" with open("README.md") as f: readme = f.read() diff --git a/tests/modules/patch.py b/tests/modules/patch.py index 95cc2cad9..338d890f2 100644 --- a/tests/modules/patch.py +++ b/tests/modules/patch.py @@ -18,7 +18,7 @@ """ ORG_SHA = "002623ccc88a3b0cb302c7d8f13792a95354d9f2" -CORRECT_SHA = "0245a9277d51a47c8aa68d264d294cf45312fab8" +CORRECT_SHA = "1dff30bfca2d98eb7ac7b09269a15e822451d99f" SUCCEED_SHA = "ba15c20c032c549d77c5773659f19c2927daf48e" FAIL_SHA = "67b642d4471c4005220a342cad3818d5ba2b5a73" BISMARK_ALIGN = "bismark/align"