From 83533409da603c1dca959296c63bbd510ff48b8a Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 6 Jun 2023 10:21:30 +0000 Subject: [PATCH 01/36] Update seqeralabs/action-tower-launch to v1.2.4 - Updates the Github action to v1.2.4 - Adds specific revision which no longer defaults to github.sha - Saves the JSON log created per pipeline as artifact --- CHANGELOG.md | 1 + .../pipeline-template/.github/workflows/awsfulltest.yml | 8 ++++++-- nf_core/pipeline-template/.github/workflows/awstest.yml | 8 ++++++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f6..a0584f3e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Updates seqeralabs/action-tower-launch to v1.2.4 ### Download diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 9511ea57b..de2fb535a 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v1.2.4 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} @@ -22,13 +22,17 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-${{ github.sha }}{% endraw %}" } profiles: test_full + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 6e1fd8c92..6e5a9a9e4 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,18 +12,22 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1 + uses: seqeralabs/action-tower-launch@v1.2.4 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { "outdir": "s3://{% raw %}${{ secrets.AWS_S3_BUCKET }}{% endraw %}/{{ short_name }}/{% raw %}results-test-${{ github.sha }}{% endraw %}" } profiles: test + - uses: actions/upload-artifact@v3 with: name: Tower debug log file - path: tower_action_*.log + path: | + tower_action_*.log + tower_action_*.json From 410db78bf8fdddb8302256a5c08a5bbb5da60fe6 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Tue, 6 Jun 2023 15:33:47 +0000 Subject: [PATCH 02/36] Update action to v2.0.0 (breaking change) --- CHANGELOG.md | 2 +- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a0584f3e3..8a1fb5ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) -- Updates seqeralabs/action-tower-launch to v1.2.4 +- Updates seqeralabs/action-tower-launch to v2.0.0 ([#2301](https://github.com/nf-core/tools/pull/2301)) ### Download diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index de2fb535a..ebad00719 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1.2.4 + uses: seqeralabs/action-tower-launch@v2.0.0 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 6e5a9a9e4..8819b7cca 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v1.2.4 + uses: seqeralabs/action-tower-launch@v2.0.0 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From e40580ba71445049caa786fb5b7d4dd639c8713b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 8 Jun 2023 08:59:02 +0000 Subject: [PATCH 03/36] Reduce to shortened version of action-tower-launch version --- nf_core/pipeline-template/.github/workflows/awsfulltest.yml | 2 +- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index ebad00719..1d5c53bda 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2.0.0 + uses: seqeralabs/action-tower-launch@v2 # TODO nf-core: You can customise AWS full pipeline tests as required # Add full size test data (but still relatively small datasets for few samples) # on the `test_full.config` test runs with only one set of parameters {%- raw %} diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index 8819b7cca..9a0bf4afb 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -12,7 +12,7 @@ jobs: steps: # Launch workflow using Tower CLI tool action {%- raw %} - name: Launch workflow via tower - uses: seqeralabs/action-tower-launch@v2.0.0 + uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} From d5b7bd220e96d030c891a307cbc30117a7bc1eca Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Jun 2023 11:19:32 +0100 Subject: [PATCH 04/36] Add singularity.registry = 'quay.io' to pipeline template --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/create-lint-wf.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- CHANGELOG.md | 2 ++ nf_core/lint/readme.py | 2 +- .../.github/ISSUE_TEMPLATE/bug_report.yml | 2 +- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- nf_core/pipeline-template/README.md | 2 +- nf_core/pipeline-template/nextflow.config | 7 ++++--- 9 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 1bd2e27fb..8fdd2bd7e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -33,7 +33,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 22.10.1)_ + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 46152d92a..654d248f7 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: # Get the repo code diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb4..7cff154a0 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f6..9572c9544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Add `singularity.registry = 'quay.io'` in pipeline template +- Bump minimum required NF version in pipeline template from `22.10.1` -> `23.04.0` ### Download diff --git a/nf_core/lint/readme.py b/nf_core/lint/readme.py index ae5c54283..55060442b 100644 --- a/nf_core/lint/readme.py +++ b/nf_core/lint/readme.py @@ -36,7 +36,7 @@ def readme(self): if "nextflow_badge" not in ignore_configs: # Check that there is a readme badge showing the minimum required version of Nextflow - # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) + # [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) # and that it has the correct version nf_badge_re = r"\[!\[Nextflow\]\(https://img\.shields\.io/badge/nextflow%20DSL2-!?(?:%E2%89%A5|%3E%3D)([\d\.]+)-23aa62\.svg\)\]\(https://www\.nextflow\.io/\)" match = re.search(nf_badge_re, content) diff --git a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml index f93cd55d5..063690f29 100644 --- a/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml @@ -42,7 +42,7 @@ body: attributes: label: System information description: | - * Nextflow version _(eg. 22.10.1)_ + * Nextflow version _(eg. 23.04.0)_ * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * Container engine: _(e.g. Docker, Singularity, Conda, Podman, Shifter, Charliecloud, or Apptainer)_ diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index bf3dc36bc..521f3e664 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" + - "23.04.0" - "latest-everything" steps: - name: Check out pipeline code diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 5ac65123e..e66746dcd 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -10,7 +10,7 @@ {%- if github_badges -%} [![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) -[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A522.10.1-23aa62.svg)](https://www.nextflow.io/) +[![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2650572c9..37137bb02 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -176,8 +176,9 @@ profiles { // Set default registry for Docker and Podman independent of -profile // Will not be used unless Docker / Podman are enabled // Set to your registry if you have a mirror of containers -docker.registry = 'quay.io' -podman.registry = 'quay.io' +singularity.registry = 'quay.io' +docker.registry = 'quay.io' +podman.registry = 'quay.io' {% if igenomes %} // Load igenomes.config if required @@ -226,7 +227,7 @@ manifest { homePage = 'https://github.com/{{ name }}' description = """{{ description }}""" mainScript = 'main.nf' - nextflowVersion = '!>=22.10.1' + nextflowVersion = '!>=23.04.0' version = '{{ version }}' doi = '' } From 114308419617c38654599d062d4befa2780f8840 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:09:46 +0100 Subject: [PATCH 05/36] fix: Parsing of container directive Changes: - Strip additional components off container directive prior to parsing - Helps parse additional strings that may be present at the front of container directive (e.g. 'container '). - Should catch more non-standard container directives (of which there are many). --- CHANGELOG.md | 1 + nf_core/modules/lint/main_nf.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2deeef5f6..d99229df9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) +- Fix parsing of container directive when it is not typical nf-core format. ### Download diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 31b8adca3..cfbc2c4ad 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -240,7 +240,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip(" '\"") + l = l.strip().removeprefix("container ").strip(" \n'\"") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) @@ -263,7 +263,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?'", l) + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -275,13 +275,13 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) - docker_tag = NoneD + docker_tag = None if l.startswith("quay.io/"): l_stripped = re.sub(r"\W+$", "", l) self.failed.append( From da476f2c76e28371ebe54eb54068369963848f95 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:16:11 +0100 Subject: [PATCH 06/36] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d99229df9..d6599be57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ - Remove `aws_tower` profile ([#2287])(https://github.com/nf-core/tools/pull/2287) - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) -- Fix parsing of container directive when it is not typical nf-core format. +- Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) ### Download From c98848a8bb088e1d6c75759fe8444c165f118348 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 13:59:10 +0100 Subject: [PATCH 07/36] Removed 'removeprefix' which is Py > 3.9 only --- nf_core/modules/lint/main_nf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index cfbc2c4ad..c6ea2ed6c 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -240,7 +240,12 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip().removeprefix("container ").strip(" \n'\"") + l = l.strip(" \n'\"") + + # Catch preceeding "container " + if l.startswith("container"): + l = l.replace("container", "").strip(" \n'\"") + if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) From 8fe15ea2301c9d391ffcfd46957b3abb392dfe5c Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 16:52:52 +0100 Subject: [PATCH 08/36] Use specific Nextflow + nf-amazon version to get check if tests pass --- .github/workflows/create-test-wf.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb4..d52fa2230 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,8 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" - - "latest-everything" + - "23.04.1" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -47,7 +46,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 - name: Upload log file artifact if: ${{ always() }} From d24652af1f7e9dd657bfe0d68b4b637b0c2125b9 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Thu, 8 Jun 2023 16:52:52 +0100 Subject: [PATCH 09/36] Use specific Nextflow + nf-amazon version to get check if tests pass --- .github/workflows/create-test-wf.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 8f3c5fdb4..d52fa2230 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,8 +23,7 @@ jobs: strategy: matrix: NXF_VER: - - "22.10.1" - - "latest-everything" + - "23.04.1" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -47,7 +46,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 - name: Upload log file artifact if: ${{ always() }} From b4f86d5bd9c052b6edde8bd2f3604dcd1c95a9f0 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Thu, 8 Jun 2023 19:55:50 +0100 Subject: [PATCH 10/36] Update create-test-wf.yml --- .github/workflows/create-test-wf.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d52fa2230..8f3c5fdb4 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,8 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.1" + - "22.10.1" + - "latest-everything" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -46,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} From 51882b2559ed06fe7c9321ff7c0601e1ee8240cc Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 09:19:05 +0100 Subject: [PATCH 11/36] Revert "Use specific Nextflow + nf-amazon version to get check if tests pass" This reverts commit d24652af1f7e9dd657bfe0d68b4b637b0c2125b9. --- .github/workflows/create-test-wf.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d52fa2230..8f3c5fdb4 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -23,7 +23,8 @@ jobs: strategy: matrix: NXF_VER: - - "23.04.1" + - "22.10.1" + - "latest-everything" steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -46,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain - nextflow run nf-core-testpipeline -profile test,docker --outdir ./results -plugins nf-amazon@1.16.2 + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} From 446a6e5fab4761c778bcf008a513b56ede5b9dce Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:11:47 +0100 Subject: [PATCH 12/36] Fix regex for containers --- nf_core/modules/lint/main_nf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index c6ea2ed6c..5025fa039 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?", l) + match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)'", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -280,7 +280,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)", l) + match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) From 32fa7707498e6073211a85947d3f70de406520df Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:28:05 +0100 Subject: [PATCH 13/36] Fix parsing of tags so they should match in most situations --- nf_core/modules/lint/main_nf.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 5025fa039..08645f56b 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -222,9 +222,6 @@ def check_process_section(self, lines, fix_version, progress_bar): return self.passed.append(("process_exist", "Process definition exists", self.main_nf)) - # Checks that build numbers of bioconda, singularity and docker container are matching - singularity_tag = "singularity" - docker_tag = "docker" bioconda_packages = [] # Process name should be all capital letters @@ -240,11 +237,11 @@ def check_process_section(self, lines, fix_version, progress_bar): # Deprecated enable_conda for i, l in enumerate(lines): url = None - l = l.strip(" \n'\"") + l = l.strip(" \n'\"}:") # Catch preceeding "container " if l.startswith("container"): - l = l.replace("container", "").strip(" \n'\"") + l = l.replace("container", "").strip(": \n'\"}") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] @@ -268,7 +265,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r"(?:/)?(?:biocontainers_)?(?::)?([A-Za-z\d\-_.]+?)(?:\.img)'", l) + match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -280,7 +277,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "docker": # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 - match = re.search(r"(?:[/])?(?::)?([A-Za-z\d\-_.]+)'", l) + match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: docker_tag = match.group(1) self.passed.append(("docker_tag", f"Found docker tag: {docker_tag}", self.main_nf)) From 90dc179cdd00097c9be1fbad4e587329a5f14eae Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:29:52 +0100 Subject: [PATCH 14/36] Match strip strings for consistency --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 08645f56b..a31318d93 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -241,7 +241,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Catch preceeding "container " if l.startswith("container"): - l = l.replace("container", "").strip(": \n'\"}") + l = l.replace("container", "").strip(" \n'\"}:") if _container_type(l) == "conda": bioconda_packages = [b for b in l.split() if "bioconda::" in b] From b6dbc8b8f2e061afde5fb310c21599f47a100610 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:37:10 +0100 Subject: [PATCH 15/36] Revert setting docker_tag and singularity_tag --- nf_core/modules/lint/main_nf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index a31318d93..85e1f9aa9 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -222,6 +222,11 @@ def check_process_section(self, lines, fix_version, progress_bar): return self.passed.append(("process_exist", "Process definition exists", self.main_nf)) + # Checks that build numbers of bioconda, singularity and docker container are matching + singularity_tag = "singularity" + docker_tag = "docker" + bioconda_packages = [] + bioconda_packages = [] # Process name should be all capital letters From 56d1a994b08d3a0be581f089afb454abfc2334f0 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 17:36:51 +0100 Subject: [PATCH 16/36] Fewer false positives for docker container names --- nf_core/modules/lint/main_nf.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 85e1f9aa9..6f2d355a7 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -223,10 +223,8 @@ def check_process_section(self, lines, fix_version, progress_bar): self.passed.append(("process_exist", "Process definition exists", self.main_nf)) # Checks that build numbers of bioconda, singularity and docker container are matching - singularity_tag = "singularity" - docker_tag = "docker" - bioconda_packages = [] - + singularity_tag = None + docker_tag = None bioconda_packages = [] # Process name should be all capital letters @@ -419,7 +417,11 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.passed.append(("bioconda_latest", f"Conda package is the latest available: `{bp}`", self.main_nf)) - return docker_tag == singularity_tag + # Check if a tag exists at all. If not, return None. + if singularity_tag is None or docker_tag is None: + return None + else: + return docker_tag == singularity_tag def check_process_labels(self, lines): @@ -608,5 +610,5 @@ def _container_type(line): if url_match: return "singularity" return None - if line.count("/") >= 1 and line.count(":") == 1: + if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0: return "docker" From 1a04dfca1713e86ba503b7bba133b93c36809d55 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 18:10:10 +0100 Subject: [PATCH 17/36] More singularity tag parsing --- nf_core/modules/lint/main_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 6f2d355a7..bddf99a06 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,7 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 - match = re.search(r":([A-Za-z\d\-_.]+)$", l) + match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) From 87144bfff9386b7c28b37f3687fcff87752ceabb Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 09:26:08 +0100 Subject: [PATCH 18/36] Update comment description of check_process_section where it extracts the container tag --- nf_core/modules/lint/main_nf.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index bddf99a06..d86566a6f 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -266,8 +266,8 @@ def check_process_section(self, lines, fix_version, progress_bar): ) ) if _container_type(l) == "singularity": - # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' :" -> v1.2.0_cv1 - # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' :" -> 0.11.9--0 + # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1 + # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0 match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) @@ -278,8 +278,8 @@ def check_process_section(self, lines, fix_version, progress_bar): url = urlparse(l.split("'")[0]) if _container_type(l) == "docker": - # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5' }" -> 2.7.1--pl526_5 - # e.g. "biocontainers/biocontainers:v1.2.0_cv1' }" -> v1.2.0_cv1 + # e.g. "quay.io/biocontainers/krona:2.7.1--pl526_5 -> 2.7.1--pl526_5 + # e.g. "biocontainers/biocontainers:v1.2.0_cv1 -> v1.2.0_cv1 match = re.search(r":([A-Za-z\d\-_.]+)$", l) if match is not None: docker_tag = match.group(1) From 6ef1fa3c5ed205510ca48467cbd56bd0a88d15e3 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 10:06:12 +0100 Subject: [PATCH 19/36] Add period to singularity tag parsing --- nf_core/modules/lint/main_nf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d86566a6f..73d1df365 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -268,7 +268,8 @@ def check_process_section(self, lines, fix_version, progress_bar): if _container_type(l) == "singularity": # e.g. "https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img -> v1.2.0_cv1 # e.g. "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0 -> 0.11.9--0 - match = re.search(r"(?::)?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) + # Please god let's find a better way to do this than regex + match = re.search(r"(?:[:.])?([A-Za-z\d\-_.]+?)(?:\.img)?(?:\.sif)?$", l) if match is not None: singularity_tag = match.group(1) self.passed.append(("singularity_tag", f"Found singularity tag: {singularity_tag}", self.main_nf)) @@ -600,7 +601,7 @@ def _container_type(line): """Returns the container type of a build.""" if line.startswith("conda"): return "conda" - if line.startswith("https://containers") or line.startswith("https://depot"): + if line.startswith("https://") or line.startswith("https://depot"): # Look for a http download URL. # Thanks Stack Overflow for the regex: https://stackoverflow.com/a/3809435/713980 url_regex = ( @@ -610,5 +611,5 @@ def _container_type(line): if url_match: return "singularity" return None - if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0: + if line.count("/") >= 1 and line.count(":") == 1 and line.count(" ") == 0 and "https://" not in line: return "docker" From 5d00842f9c4ffd97c1a846f0ce95324b5b02b702 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Fri, 9 Jun 2023 12:09:33 +0100 Subject: [PATCH 20/36] Add ability to select custom registry when linting modules Changes: - Adds --registry parameter to nf-core modules lint - Will check container definition via this registry - Default to quay.io --- CHANGELOG.md | 1 + nf_core/__main__.py | 6 +++++- nf_core/modules/lint/__init__.py | 16 +++++++++------- nf_core/modules/lint/main_nf.py | 24 ++++++++++++++++-------- tests/modules/lint.py | 10 ++++++++++ 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6599be57..407e04ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) +- Add ability to specify custom registry for linting modules, defaults to quay.io ### Download diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 6d6ded471..00fba3631 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -807,6 +807,9 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") +@click.option( + "-r", "--registry", type=str, metavar="", default="quay.io", help="Registry to use for containers" +) @click.option("-k", "--key", type=str, metavar="", multiple=True, help="Run only these lint tests") @click.option("-a", "--all", is_flag=True, help="Run on all modules") @click.option("-w", "--fail-warned", is_flag=True, help="Convert warn tests to failures") @@ -821,7 +824,7 @@ def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): ) @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") def lint( - ctx, tool, dir, key, all, fail_warned, local, passed, sort_by, fix_version + ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version ): # pylint: disable=redefined-outer-name """ Lint one or more modules in a directory. @@ -846,6 +849,7 @@ def lint( ) module_lint.lint( module=tool, + registry=registry, key=key, all_modules=all, print_results=True, diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 24d673b1c..f288f1776 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -145,6 +145,7 @@ def get_all_lint_tests(is_pipeline): def lint( self, module=None, + registry="quay.io", key=(), all_modules=False, print_results=True, @@ -227,11 +228,11 @@ def lint( # Lint local modules if local and len(local_modules) > 0: - self.lint_modules(local_modules, local=True, fix_version=fix_version) + self.lint_modules(local_modules, registry=registry, local=True, fix_version=fix_version) # Lint nf-core modules if len(remote_modules) > 0: - self.lint_modules(remote_modules, local=False, fix_version=fix_version) + self.lint_modules(remote_modules, registry=registry, local=False, fix_version=fix_version) if print_results: self._print_results(show_passed=show_passed, sort_by=sort_by) @@ -264,12 +265,13 @@ def filter_tests_by_key(self, key): # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in key] - def lint_modules(self, modules, local=False, fix_version=False): + def lint_modules(self, modules, registry, local=False, fix_version=False): """ Lint a list of modules Args: modules ([NFCoreModule]): A list of module objects + registry (str): The container registry to use. Should be quay.io in most situations. local (boolean): Whether the list consist of local or nf-core modules fix_version (boolean): Fix the module version if a newer version is available """ @@ -290,9 +292,9 @@ def lint_modules(self, modules, local=False, fix_version=False): for mod in modules: progress_bar.update(lint_progress, advance=1, test_name=mod.module_name) - self.lint_module(mod, progress_bar, local=local, fix_version=fix_version) + self.lint_module(mod, progress_bar, registry=registry, local=local, fix_version=fix_version) - def lint_module(self, mod, progress_bar, local=False, fix_version=False): + def lint_module(self, mod, progress_bar, registry, local=False, fix_version=False): """ Perform linting on one module @@ -311,7 +313,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): # Only check the main script in case of a local module if local: - self.main_nf(mod, fix_version, progress_bar) + self.main_nf(mod, fix_version, registry, progress_bar) self.passed += [LintResult(mod, *m) for m in mod.passed] warned = [LintResult(mod, *m) for m in (mod.warned + mod.failed)] if not self.fail_warned: @@ -323,7 +325,7 @@ def lint_module(self, mod, progress_bar, local=False, fix_version=False): else: for test_name in self.lint_tests: if test_name == "main_nf": - getattr(self, test_name)(mod, fix_version, progress_bar) + getattr(self, test_name)(mod, fix_version, registry, progress_bar) else: getattr(self, test_name)(mod) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 73d1df365..18d95bd37 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -17,7 +17,7 @@ log = logging.getLogger(__name__) -def main_nf(module_lint_object, module, fix_version, progress_bar): +def main_nf(module_lint_object, module, fix_version, registry, progress_bar): """ Lint a ``main.nf`` module file @@ -121,7 +121,7 @@ def main_nf(module_lint_object, module, fix_version, progress_bar): module.passed.append(("main_nf_script_outputs", "Process 'output' block found", module.main_nf)) # Check the process definitions - if check_process_section(module, process_lines, fix_version, progress_bar): + if check_process_section(module, process_lines, registry, fix_version, progress_bar): module.passed.append(("main_nf_container", "Container versions match", module.main_nf)) else: module.warned.append(("main_nf_container", "Container versions do not match", module.main_nf)) @@ -209,12 +209,20 @@ def check_when_section(self, lines): self.passed.append(("when_condition", "when: condition is unchanged", self.main_nf)) -def check_process_section(self, lines, fix_version, progress_bar): - """ - Lint the section of a module between the process definition +def check_process_section(self, lines, registry, fix_version, progress_bar): + """Lint the section of a module between the process definition and the 'input:' definition Specifically checks for correct software versions and containers + + Args: + lines (List[str]): Content of process. + registry (str): Base Docker registry for containers. Typically quay.io. + fix_version (bool): Fix software version + progress_bar (ProgressBar): Progress bar to update. + + Returns: + Optional[bool]: True if singularity and docker containers match, False otherwise. If process definition does not exist, None. """ # Check that we have a process section if len(lines) == 0: @@ -288,7 +296,7 @@ def check_process_section(self, lines, fix_version, progress_bar): else: self.failed.append(("docker_tag", "Unable to parse docker tag", self.main_nf)) docker_tag = None - if l.startswith("quay.io/"): + if l.startswith(registry): l_stripped = re.sub(r"\W+$", "", l) self.failed.append( ( @@ -303,7 +311,7 @@ def check_process_section(self, lines, fix_version, progress_bar): # Guess if container name is simple one (e.g. nfcore/ubuntu:20.04) # If so, add quay.io as default container prefix if l.count("/") == 1 and l.count(":") == 1: - l = "quay.io/" + l + l = "/".join([registry, l]).replace("//", "/") url = urlparse(l.split("'")[0]) # lint double quotes @@ -326,7 +334,7 @@ def check_process_section(self, lines, fix_version, progress_bar): ) # lint more than one container in the same line - if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or "quay.io/" in l): + if ("https://containers" in l or "https://depot" in l) and ("biocontainers/" in l or l.startswith(registry)): self.warned.append( ( "container_links", diff --git a/tests/modules/lint.py b/tests/modules/lint.py index b7aaf610c..8e692e8ca 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -82,6 +82,16 @@ def test_modules_lint_multiple_remotes(self): assert len(module_lint.warned) >= 0 +def test_modules_lint_registry(self): + """Test linting the TrimGalore! module""" + self.mods_install.install("samtools") + module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) + module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + def test_modules_lint_patched_modules(self): """ Test creating a patch file and applying it to a new version of the the files From 9b70efb5d1ce2d64ac770bebdff8c4bef827d1ec Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 10:34:01 +0100 Subject: [PATCH 21/36] Quick test update --- tests/modules/lint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 8e692e8ca..66da0019d 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -83,7 +83,7 @@ def test_modules_lint_multiple_remotes(self): def test_modules_lint_registry(self): - """Test linting the TrimGalore! module""" + """Test linting the samtools module and alternative registry""" self.mods_install.install("samtools") module_lint = nf_core.modules.ModuleLint(dir=self.pipeline_dir) module_lint.lint(print_results=False, registry="public.ecr.aws", module="samtools") From 464f0f75856ecedf3776d45c4d11a2d6ef14d3d5 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 11:04:02 +0100 Subject: [PATCH 22/36] CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 407e04ed4..be0ab9828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Fixed the Slack report to include the pipeline name ([#2291](https://github.com/nf-core/tools/pull/2291)) - Fix link in the MultiQC report to point to exact version of output docs ([#2298](https://github.com/nf-core/tools/pull/2298)) - Fix parsing of container directive when it is not typical nf-core format ([#2306](https://github.com/nf-core/tools/pull/2306)) -- Add ability to specify custom registry for linting modules, defaults to quay.io +- Add ability to specify custom registry for linting modules, defaults to quay.io ([#2313](https://github.com/nf-core/tools/pull/2313)) ### Download From 5e52513f701540c78828d0c769a445e658b4cd21 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 11:04:30 +0100 Subject: [PATCH 23/36] Added 'quay.io' as default for lint_modules --- nf_core/modules/lint/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index f288f1776..ba3b3d18f 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -265,7 +265,7 @@ def filter_tests_by_key(self, key): # If -k supplied, only run these tests self.lint_tests = [k for k in self.lint_tests if k in key] - def lint_modules(self, modules, registry, local=False, fix_version=False): + def lint_modules(self, modules, registry="quay.io", local=False, fix_version=False): """ Lint a list of modules From 4fe8a043d686199240e16dd393d050626ed846a7 Mon Sep 17 00:00:00 2001 From: Adam Talbot Date: Mon, 12 Jun 2023 13:28:38 +0100 Subject: [PATCH 24/36] Add test for quay.io and public.aws.ecr explicitly for a direct comparison --- tests/modules/lint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 66da0019d..5b2c25080 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -90,6 +90,10 @@ def test_modules_lint_registry(self): assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + module_lint.lint(print_results=False, registry="quay.io", module="samtools") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 def test_modules_lint_patched_modules(self): From e5716819de7d959377ad29b048a686a156066475 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 12 Jun 2023 15:49:29 +0200 Subject: [PATCH 25/36] add docs to commit new files before pytest --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index dacb50ebc..5d5791c10 100644 --- a/README.md +++ b/README.md @@ -941,6 +941,8 @@ before_command: sed 's/1.13a/1.10/g' modules/multiqc/main.nf > modules/multiqc/m To run unit tests of a module that you have installed or the test created by the command [`nf-core modules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software///test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). +> This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new module to add the new files to your git index. + You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts.