From 528367785c95abbebd638dd2c4b5465ba8610909 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 14:04:53 +0200 Subject: [PATCH 001/158] generalize nextflow_cmd to run_cmd to use the same structure for nf-test --- nf_core/list.py | 8 ++++---- nf_core/utils.py | 18 +++++++++++------- tests/test_download.py | 6 +++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/nf_core/list.py b/nf_core/list.py index 2bb90cb00e..c6c4bdf756 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -64,10 +64,10 @@ def get_local_wf(workflow, revision=None): # Wasn't local, fetch it log.info(f"Downloading workflow: {workflow} ({revision})") - pull_cmd = f"nextflow pull {workflow}" + pull_cmd = f"pull {workflow}" if revision is not None: pull_cmd += f" -r {revision}" - nf_core.utils.nextflow_cmd(pull_cmd) + nf_core.utils.run_cmd("nextflow", pull_cmd) local_wf = LocalWorkflow(workflow) local_wf.get_local_nf_workflow_details() return local_wf.local_path @@ -128,7 +128,7 @@ def get_local_nf_workflows(self): # Fetch details about local cached pipelines with `nextflow list` else: log.debug("Getting list of local nextflow workflows") - nflist_raw = nf_core.utils.nextflow_cmd("nextflow list") + nflist_raw = nf_core.utils.run_cmd("nextflow", "list") for wf_name in nflist_raw.splitlines(): if not str(wf_name).startswith("nf-core/"): self.local_unmatched.append(wf_name) @@ -342,7 +342,7 @@ def get_local_nf_workflow_details(self): # Use `nextflow info` to get more details about the workflow else: - nfinfo_raw = str(nf_core.utils.nextflow_cmd(f"nextflow info -d {self.full_name}")) + nfinfo_raw = str(nf_core.utils.run_cmd("nextflow", f"info -d {self.full_name}")) re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} for key, pattern in re_patterns.items(): m = re.search(pattern, nfinfo_raw) diff --git a/nf_core/utils.py b/nf_core/utils.py index 3f7d9693b1..3e5581f4b1 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -17,6 +17,7 @@ import sys import time from pathlib import Path +from typing import ByteString, Optional, Union import git import prompt_toolkit @@ -269,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = nextflow_cmd(f"nextflow config -flat {wf_path}") + nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}") for l in nfconfig_raw.splitlines(): ul = l.decode("utf-8") try: @@ -303,17 +304,20 @@ def fetch_wf_config(wf_path, cache_config=True): return config -def nextflow_cmd(cmd): - """Run a Nextflow command and capture the output. Handle errors nicely""" +def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]: + """Run a specified command and capture the output. Handle errors nicely.""" + full_cmd = f"{executable} {cmd}" try: - nf_proc = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - return nf_proc.stdout + proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return proc.stdout except OSError as e: if e.errno == errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + raise AssertionError( + f"It looks like {executable} is not installed. Please ensure it is available in your PATH." + ) except subprocess.CalledProcessError as e: raise AssertionError( - f"Command '{cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" + f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" ) diff --git a/tests/test_download.py b/tests/test_download.py index ee0744f660..9416d462e4 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -16,7 +16,7 @@ import nf_core.utils from nf_core.download import ContainerError, DownloadWorkflow, WorkflowRepo from nf_core.synced_repo import SyncedRepo -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, nextflow_cmd +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, run_cmd from .utils import with_temporary_file, with_temporary_folder @@ -156,8 +156,8 @@ def test_find_container_images_config_basic(self, tmp_path, mock_fetch_wf_config @mock.patch("nf_core.utils.fetch_wf_config") def test__find_container_images_config_nextflow(self, tmp_path, mock_fetch_wf_config): download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path) - nfconfig_raw = nextflow_cmd( - f"nextflow config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}" + nfconfig_raw = run_cmd( + "nextflow", f"config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}" ) config = {} for l in nfconfig_raw.splitlines(): From 00817668dd2c3cba1b6f1d685eab09ea7f288b35 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 15:03:57 +0200 Subject: [PATCH 002/158] initial changes for nf-test --- nf_core/components/create.py | 102 ++++++++---------- nf_core/module-template/tests/main.nf | 18 ---- nf_core/module-template/tests/nextflow.config | 5 - nf_core/module-template/tests/test.yml | 18 ---- 4 files changed, 46 insertions(+), 97 deletions(-) delete mode 100644 nf_core/module-template/tests/main.nf delete mode 100644 nf_core/module-template/tests/nextflow.config delete mode 100644 nf_core/module-template/tests/test.yml diff --git a/nf_core/components/create.py b/nf_core/components/create.py index e626de4aaa..dbc7f90ff5 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -10,11 +10,12 @@ import os import re import subprocess +from pathlib import Path +from typing import Optional import jinja2 import questionary import rich -import yaml from packaging.version import parse as parse_version import nf_core @@ -27,16 +28,16 @@ class ComponentCreate(ComponentCommand): def __init__( self, - component_type, - directory=".", - component="", - author=None, - process_label=None, - has_meta=None, - force=False, - conda_name=None, - conda_version=None, - empty_template=False, + component_type: str, + directory: str = ".", + component: str = "", + author: Optional[str] = None, + process_label: Optional[str] = None, + has_meta: Optional[str] = None, + force: bool = False, + conda_name: Optional[str] = None, + conda_version: Optional[str] = None, + empty_template: bool = False, ): super().__init__(component_type, directory) self.directory = directory @@ -71,23 +72,23 @@ def create(self): If is a pipeline, this function creates a file called: '/modules/local/tool.nf' - OR + OR '/modules/local/tool_subtool.nf' - OR for subworkflows + OR for subworkflows '/subworkflows/local/subworkflow_name.nf' If is a clone of nf-core/modules, it creates or modifies the following files: For modules: + ```tree modules/modules/nf-core/tool/subtool/ - * main.nf - * meta.yml - modules/tests/modules/nf-core/tool/subtool/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml + ├── main.nf + ├── meta.yml + └── tests + ├── main.nf.test + └── tags.yml + ``` The function will attempt to automatically find a Bioconda package called and matching Docker / Singularity images from BioContainers. @@ -147,32 +148,23 @@ def create(self): # Create component template with jinja2 self._render_template() - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - if self.subtool: - pytest_modules_yml[self.component_name] = [ - f"modules/{self.org}/{self.component}/{self.subtool}/**", - f"tests/modules/{self.org}/{self.component}/{self.subtool}/**", - ] + # generate nf-tests + nf_core.utils.run_cmd("nf-test", f"generate process {self.file_paths['main.nf']}") + # move generated main.nf.test file into test directory + + Path("main.nf.test").rename(self.file_paths[os.path.join(self.component_type, "test", "main.nf.test")]) + + # inside main.nf.test replace the path to the main.nf file with the relative path for the script + with open(self.file_paths["tests/main.nf.test"], "r") as f: + lines = f.readlines() + with open(self.file_paths["tests/main.nf.test"], "w") as f: + for line in lines: + if line.startswith("script"): + f.write(f"script ../main.nf\n") else: - pytest_modules_yml[ - ("" if self.component_type == "modules" else self.component_type + "/") + self.component_name - ] = [ - f"{self.component_type}/{self.org}/{self.component}/**", - f"tests/{self.component_type}/{self.org}/{self.component}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + f.write(line) new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) log.info("Created / edited following files:\n " + "\n ".join(new_files)) def _get_bioconda_tool(self): @@ -362,17 +354,12 @@ def _get_component_dirs(self): file_paths[os.path.join(self.component_type, "main.nf")] = component_file if self.repo_type == "modules": - software_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) - test_dir = os.path.join(self.directory, "tests", self.component_type, self.org, self.component_dir) + component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(software_dir) and not self.force_overwrite: + if os.path.exists(component_dir) and not self.force_overwrite: raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite" - ) - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite" + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) if self.component_type == "modules": @@ -403,11 +390,14 @@ def _get_component_dirs(self): # Set file paths # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") + file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") + file_paths[os.path.join(self.component_type, "test", "main.nf.test")] = os.path.join( + component_dir, "test", "main.nf.test" + ) + file_paths[os.path.join(self.component_type, "test", "tags.yml")] = os.path.join( + component_dir, "test", "tags.yml" + ) return file_paths diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf deleted file mode 100644 index fcb7195fe4..0000000000 --- a/nf_core/module-template/tests/main.nf +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { {{ component_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ component_dir }}/main.nf' - -workflow test_{{ component_name_underscore }} { - {% if has_meta %} - input = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - {%- else %} - input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) - {%- endif %} - - {{ component_name_underscore|upper }} ( input ) -} diff --git a/nf_core/module-template/tests/nextflow.config b/nf_core/module-template/tests/nextflow.config deleted file mode 100644 index 50f50a7a35..0000000000 --- a/nf_core/module-template/tests/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} \ No newline at end of file diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml deleted file mode 100644 index a2cedb73d3..0000000000 --- a/nf_core/module-template/tests/test.yml +++ /dev/null @@ -1,18 +0,0 @@ -{%- if not_empty_template -%} -## TODO nf-core: Please run the following command to build this file: -# nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{% endif -%} -- name: "{{ component }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config - tags: - - "{{ component }}{% if subtool -%}" - - "{{ component }}/{{ subtool }}{%- endif %}" - files: - {% if not_empty_template -%} - - path: "output/{{ component }}/test.bam" - md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: "output/{{ component }}/versions.yml" - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - {% else -%} - - path: "" - {%- endif %} From d9126e5b93988c38a4af7cd32871b0c6f0829d87 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 15:24:10 +0200 Subject: [PATCH 003/158] move tags.yml to correct position --- nf_core/module-template/modules/tests/tags.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 nf_core/module-template/modules/tests/tags.yml diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/modules/tests/tags.yml new file mode 100644 index 0000000000..93f8228378 --- /dev/null +++ b/nf_core/module-template/modules/tests/tags.yml @@ -0,0 +1,2 @@ +{ { component_name_underscore } }: + - "modules/{{ org }}/{{ component_dir }}/**" From ecda68767b2ac46da81e11747ad6af9bbdcb03c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 24 Oct 2023 16:05:11 +0200 Subject: [PATCH 004/158] Update nf_core/utils.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 3e5581f4b1..09ad808c98 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -270,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}") + nfconfig_raw = run_cmd("nextflow", f"config -flat {wf_path}") for l in nfconfig_raw.splitlines(): ul = l.decode("utf-8") try: From 955321b01145a6ef7c2192a568d332a47668fd19 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 16:23:53 +0200 Subject: [PATCH 005/158] convert all cmds --- nf_core/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 3e5581f4b1..ddbc6ee9d0 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -270,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}") + nfconfig_raw = run_cmd("nextflow", f"config -flat {wf_path}") for l in nfconfig_raw.splitlines(): ul = l.decode("utf-8") try: @@ -307,6 +307,7 @@ def fetch_wf_config(wf_path, cache_config=True): def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]: """Run a specified command and capture the output. Handle errors nicely.""" full_cmd = f"{executable} {cmd}" + log.debug(f"Running command: {full_cmd}") try: proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) return proc.stdout From 0a4afacbff13b2241e629c59da63fad07984bfef Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 16:32:19 +0200 Subject: [PATCH 006/158] first running version --- nf_core/components/create.py | 59 +++++++++++++++--------- nf_core/module-template/modules/meta.yml | 4 +- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index dbc7f90ff5..fe08391822 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -147,23 +147,45 @@ def create(self): # Create component template with jinja2 self._render_template() + log.info(f"Created component template: '{self.component_name}'") # generate nf-tests - nf_core.utils.run_cmd("nf-test", f"generate process {self.file_paths['main.nf']}") - # move generated main.nf.test file into test directory + nf_core.utils.run_cmd( + "nf-test", f"generate process {self.file_paths[os.path.join(self.component_type, 'main.nf')]}" + ) + + # move generated main.nf.test file into tests directory + tests_dir = Path(self.directory, self.component_type, self.org, self.component_dir, "tests") + tests_dir.mkdir(exist_ok=True) + + Path(tests_dir.parent, "main.nf.test").rename(tests_dir / "main.nf.test") - Path("main.nf.test").rename(self.file_paths[os.path.join(self.component_type, "test", "main.nf.test")]) + self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( + self.directory, self.component_type, self.org, self.component_dir, "tests", "main.nf.test" + ) + log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") # inside main.nf.test replace the path to the main.nf file with the relative path for the script - with open(self.file_paths["tests/main.nf.test"], "r") as f: + with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: lines = f.readlines() - with open(self.file_paths["tests/main.nf.test"], "w") as f: + with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: for line in lines: - if line.startswith("script"): - f.write(f"script ../main.nf\n") + if line.strip().startswith("script "): + log.debug(f"Replacing script path in nf-test: '{line.strip()}'") + f.write(f" script ../main.nf\n") + # add tag elements after process line + if line.strip().startswith("process "): + log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") + f.write(line) + f.write("\n") + f.write(f" tag 'modules'\n") + f.write(f" tag 'modules_nfcore'\n") + f.write(f" tag '{self.component_name_underscore}'\n") + f.write(f" tag '{self.component}'\n") + if self.subtool: + f.write(f" tag '{self.subtool}'\n") else: f.write(line) - new_files = list(self.file_paths.values()) log.info("Created / edited following files:\n " + "\n ".join(new_files)) @@ -265,9 +287,14 @@ def _render_template(self): log.debug(f"Rendering template file: '{template_fn}'") j_template = env.get_template(template_fn) object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) + try: + rendered_output = j_template.render(object_attrs) + except Exception as e: + log.error(f"Could not render template file '{template_fn}':\n{e}") + raise e # Write output to the target file + log.debug(f"Writing output to: '{dest_fn}'") os.makedirs(os.path.dirname(dest_fn), exist_ok=True) with open(dest_fn, "w") as fh: log.debug(f"Writing output to: '{dest_fn}'") @@ -367,17 +394,10 @@ def _get_component_dirs(self): parent_tool_main_nf = os.path.join( self.directory, self.component_type, self.org, self.component, "main.nf" ) - parent_tool_test_nf = os.path.join( - self.directory, self.component_type, self.org, self.component, "main.nf" - ) if self.subtool and os.path.exists(parent_tool_main_nf): raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and os.path.exists(parent_tool_test_nf): - raise UserWarning( - f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.component_name}'" - ) # If no subtool, check that there isn't already a tool/subtool tool_glob = glob.glob( @@ -392,11 +412,8 @@ def _get_component_dirs(self): # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") - file_paths[os.path.join(self.component_type, "test", "main.nf.test")] = os.path.join( - component_dir, "test", "main.nf.test" - ) - file_paths[os.path.join(self.component_type, "test", "tags.yml")] = os.path.join( - component_dir, "test", "tags.yml" + file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( + component_dir, "tests", "tags.yml" ) return file_paths diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index aea3c36aa3..c5d9c042b4 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -30,7 +30,7 @@ input: type: map description: | Groovy Map containing sample information - e.g. `[ id:'test', single_end:false ]` + e.g. `[ id:'sample1', single_end:false ]` {% endif %} {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input @@ -49,7 +49,7 @@ output: type: map description: | Groovy Map containing sample information - e.g. `[ id:'test', single_end:false ]` + e.g. `[ id:'sample1', single_end:false ]` {% endif %} - versions: type: file From 86613b6792041fc18857d87583cf9b5712df7050 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 17:24:49 +0200 Subject: [PATCH 007/158] replace params --- nf_core/components/create.py | 53 ++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index fe08391822..b1c94566d8 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -11,7 +11,7 @@ import re import subprocess from pathlib import Path -from typing import Optional +from typing import Dict, Optional import jinja2 import questionary @@ -57,7 +57,7 @@ def __init__( self.bioconda = None self.singularity_container = None self.docker_container = None - self.file_paths = {} + self.file_paths: Dict[str, str] = {} self.not_empty_template = not empty_template def create(self): @@ -165,27 +165,40 @@ def create(self): ) log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") - # inside main.nf.test replace the path to the main.nf file with the relative path for the script + # Read the content from the file with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: lines = f.readlines() + + # Construct the modified content + modified_content = [] + for line in lines: + if line.strip().startswith("script "): + log.debug(f"Replacing script path in nf-test: '{line.strip()}'") + modified_content.append(f" script ../main.nf\n") + elif line.strip().startswith('process "'): + log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") + modified_content.append(line) + modified_content.append("\n") + modified_content.append(f" tag 'modules'\n") + modified_content.append(f" tag 'modules_nfcore'\n") + modified_content.append(f" tag '{self.component_name_underscore}'\n") + modified_content.append(f" tag '{self.component}'\n") + if self.subtool: + modified_content.append(f" tag '{self.subtool}'\n") + else: + modified_content.append(line) + + # Combine the modified content into one string + content = "".join(modified_content) + + # Apply the regex substitution + replacement = ' outdir = "$outputDir"' + content = re.sub(r"(params\s*{)\s*[^}]+(})", r"\1\n " + replacement + r"\2", content, flags=re.DOTALL) + + # Write the final content back to the file with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: - for line in lines: - if line.strip().startswith("script "): - log.debug(f"Replacing script path in nf-test: '{line.strip()}'") - f.write(f" script ../main.nf\n") - # add tag elements after process line - if line.strip().startswith("process "): - log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") - f.write(line) - f.write("\n") - f.write(f" tag 'modules'\n") - f.write(f" tag 'modules_nfcore'\n") - f.write(f" tag '{self.component_name_underscore}'\n") - f.write(f" tag '{self.component}'\n") - if self.subtool: - f.write(f" tag '{self.subtool}'\n") - else: - f.write(line) + f.write(content) + new_files = list(self.file_paths.values()) log.info("Created / edited following files:\n " + "\n ".join(new_files)) From e0c30cdca44343f5980fc8c9f5145ab01726de8a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 17:25:05 +0200 Subject: [PATCH 008/158] fix typing errors --- nf_core/utils.py | 5 +++-- requirements-dev.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index ddbc6ee9d0..6a32a45a33 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -54,10 +54,10 @@ ) NFCORE_CACHE_DIR = os.path.join( - os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME"), ".cache")), + os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME") or "", ".cache")), "nfcore", ) -NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") +NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore") def fetch_remote_version(source_url): @@ -329,6 +329,7 @@ def setup_nfcore_dir(): """ if not os.path.exists(NFCORE_DIR): os.makedirs(NFCORE_DIR) + return True def setup_requests_cachedir(): diff --git a/requirements-dev.txt b/requirements-dev.txt index 0f650b9e5c..c94874b193 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,4 @@ Sphinx sphinx-rtd-theme mypy types-PyYAML +types-requests From a9f55badfe0c28ccba5d45cd980d7556e90066c1 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Oct 2023 17:15:25 +0200 Subject: [PATCH 009/158] add TODO comments to main.nf.test and fix tags.yml template rendering --- nf_core/components/create.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index b1c94566d8..9d5f5e270d 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -172,19 +172,44 @@ def create(self): # Construct the modified content modified_content = [] for line in lines: - if line.strip().startswith("script "): + if line.strip().startswith("nextflow_process"): + log.debug("Adding TODO nf-core comment to nf-test") + modified_content.append( + "// TODO nf-core: Once you have added the required tests, please run the following command to build this file:\n" + ) + modified_content.append(f"// nf-core {self.component_type} create-test-snap {self.component_name}\n") + modified_content.append(line) + elif line.strip().startswith("script "): log.debug(f"Replacing script path in nf-test: '{line.strip()}'") modified_content.append(f" script ../main.nf\n") elif line.strip().startswith('process "'): log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") modified_content.append(line) modified_content.append("\n") - modified_content.append(f" tag 'modules'\n") - modified_content.append(f" tag 'modules_nfcore'\n") - modified_content.append(f" tag '{self.component_name_underscore}'\n") - modified_content.append(f" tag '{self.component}'\n") + modified_content.append(" tag 'modules'\n") + modified_content.append(" tag 'modules_nfcore'\n") + modified_content.append(f" tag '{self.component_name}'\n") if self.subtool: + modified_content.append(f" tag '{self.component}'\n") modified_content.append(f" tag '{self.subtool}'\n") + elif line.strip().startswith('test("Should run without failures")'): + log.debug("Adding TODO nf-core comment to change test name") + modified_content.append( + " //TODO nf-core: Change the test name preferably indicating the test-data and file-format used\n" + ) + modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') + modified_content.append(line) + elif line.strip().startswith("assert process.success"): + log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") + modified_content.append(" assertAll(\n") + modified_content.append(" { assert process.success },\n") + elif line.strip().startswith("assert snapshot(process.out).match()"): + log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") + modified_content.append(" { assert snapshot(process.out).match() }\n") + modified_content.append( + " //TODO nf-core: Add all required assertions to verify the test output.\n" + ) + modified_content.append(" )\n") else: modified_content.append(line) From 865d61328901d2c5c5df86e9da7c8571a5213e26 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Oct 2023 17:24:14 +0200 Subject: [PATCH 010/158] add todo comment for chained modules --- nf_core/components/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 9d5f5e270d..4e8bd5f1d4 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -199,6 +199,13 @@ def create(self): ) modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') modified_content.append(line) + log.debug("Adding TODO nf-core comment about using a setup method") + modified_content.append( + "\n //TODO nf-core: If you are created a test for a chained module\n" + " //(the module requires running more than one process to generate the required output)\n" + " //add the 'setup' method here.\n" + " //You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" + ) elif line.strip().startswith("assert process.success"): log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") modified_content.append(" assertAll(\n") From d767c78eb4be89d8d8ba356c9985cf0be8805f15 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:04:20 +0200 Subject: [PATCH 011/158] add real example to main.nf.test --- nf_core/components/create.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 4e8bd5f1d4..e55daa562c 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -195,17 +195,30 @@ def create(self): elif line.strip().startswith('test("Should run without failures")'): log.debug("Adding TODO nf-core comment to change test name") modified_content.append( - " //TODO nf-core: Change the test name preferably indicating the test-data and file-format used\n" + " // TODO nf-core: Change the test name preferably indicating the test-data and file-format used. Example:\n" ) - modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') - modified_content.append(line) + modified_content.append(' test("sarscov2 - bam") {\n') log.debug("Adding TODO nf-core comment about using a setup method") modified_content.append( - "\n //TODO nf-core: If you are created a test for a chained module\n" - " //(the module requires running more than one process to generate the required output)\n" - " //add the 'setup' method here.\n" - " //You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" + "\n // TODO nf-core: If you are created a test for a chained module\n" + " // (the module requires running more than one process to generate the required output)\n" + " // add the 'setup' method here.\n" + " // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" ) + elif line.strip().startswith("// input[0]"): + log.debug(f"Replacing input data example: '{line.strip()}'") + if self.has_meta: + modified_content.append( + " input = [\n" + " [ id:'test', single_end:false ], // meta map\n" + " file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)\n" + " ]\n" + ) + else: + modified_content.append( + " // input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)\n" + ) + elif line.strip().startswith("assert process.success"): log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") modified_content.append(" assertAll(\n") From 876eab39ea968de6d2af76a96f1d521e335d8c2e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:16:59 +0200 Subject: [PATCH 012/158] add tags.yml to prettierignore --- .prettierignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prettierignore b/.prettierignore index 4cd77bb4ed..1ecc7d3ada 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,4 @@ slackreport.json docs/api/_build testing nf_core/module-template/modules/meta.yml -nf_core/module-template/tests/test.yml +nf-core/module-template/modules/tests/tags.yml From 3b9f03132876f8370824ddd7f3bf6bbe332dde27 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:24:02 +0200 Subject: [PATCH 013/158] add main.nf.test to template instead of using nf-test --- .prettierignore | 2 +- nf_core/components/create.py | 100 +----------------- .../modules/tests/main.nf.test | 53 ++++++++++ .../module-template/modules/tests/tags.yml | 2 +- 4 files changed, 59 insertions(+), 98 deletions(-) create mode 100644 nf_core/module-template/modules/tests/main.nf.test diff --git a/.prettierignore b/.prettierignore index 1ecc7d3ada..9f16831c4a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,4 @@ slackreport.json docs/api/_build testing nf_core/module-template/modules/meta.yml -nf-core/module-template/modules/tests/tags.yml +nf_core/module-template/modules/tests/tags.yml diff --git a/nf_core/components/create.py b/nf_core/components/create.py index e55daa562c..8b96a02076 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -149,103 +149,8 @@ def create(self): self._render_template() log.info(f"Created component template: '{self.component_name}'") - # generate nf-tests - nf_core.utils.run_cmd( - "nf-test", f"generate process {self.file_paths[os.path.join(self.component_type, 'main.nf')]}" - ) - - # move generated main.nf.test file into tests directory - tests_dir = Path(self.directory, self.component_type, self.org, self.component_dir, "tests") - tests_dir.mkdir(exist_ok=True) - - Path(tests_dir.parent, "main.nf.test").rename(tests_dir / "main.nf.test") - - self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( - self.directory, self.component_type, self.org, self.component_dir, "tests", "main.nf.test" - ) - - log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") - # Read the content from the file - with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: - lines = f.readlines() - - # Construct the modified content - modified_content = [] - for line in lines: - if line.strip().startswith("nextflow_process"): - log.debug("Adding TODO nf-core comment to nf-test") - modified_content.append( - "// TODO nf-core: Once you have added the required tests, please run the following command to build this file:\n" - ) - modified_content.append(f"// nf-core {self.component_type} create-test-snap {self.component_name}\n") - modified_content.append(line) - elif line.strip().startswith("script "): - log.debug(f"Replacing script path in nf-test: '{line.strip()}'") - modified_content.append(f" script ../main.nf\n") - elif line.strip().startswith('process "'): - log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") - modified_content.append(line) - modified_content.append("\n") - modified_content.append(" tag 'modules'\n") - modified_content.append(" tag 'modules_nfcore'\n") - modified_content.append(f" tag '{self.component_name}'\n") - if self.subtool: - modified_content.append(f" tag '{self.component}'\n") - modified_content.append(f" tag '{self.subtool}'\n") - elif line.strip().startswith('test("Should run without failures")'): - log.debug("Adding TODO nf-core comment to change test name") - modified_content.append( - " // TODO nf-core: Change the test name preferably indicating the test-data and file-format used. Example:\n" - ) - modified_content.append(' test("sarscov2 - bam") {\n') - log.debug("Adding TODO nf-core comment about using a setup method") - modified_content.append( - "\n // TODO nf-core: If you are created a test for a chained module\n" - " // (the module requires running more than one process to generate the required output)\n" - " // add the 'setup' method here.\n" - " // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" - ) - elif line.strip().startswith("// input[0]"): - log.debug(f"Replacing input data example: '{line.strip()}'") - if self.has_meta: - modified_content.append( - " input = [\n" - " [ id:'test', single_end:false ], // meta map\n" - " file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)\n" - " ]\n" - ) - else: - modified_content.append( - " // input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)\n" - ) - - elif line.strip().startswith("assert process.success"): - log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") - modified_content.append(" assertAll(\n") - modified_content.append(" { assert process.success },\n") - elif line.strip().startswith("assert snapshot(process.out).match()"): - log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") - modified_content.append(" { assert snapshot(process.out).match() }\n") - modified_content.append( - " //TODO nf-core: Add all required assertions to verify the test output.\n" - ) - modified_content.append(" )\n") - else: - modified_content.append(line) - - # Combine the modified content into one string - content = "".join(modified_content) - - # Apply the regex substitution - replacement = ' outdir = "$outputDir"' - content = re.sub(r"(params\s*{)\s*[^}]+(})", r"\1\n " + replacement + r"\2", content, flags=re.DOTALL) - - # Write the final content back to the file - with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: - f.write(content) - new_files = list(self.file_paths.values()) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) + log.info("Created following files:\n " + "\n ".join(new_files)) def _get_bioconda_tool(self): """ @@ -473,6 +378,9 @@ def _get_component_dirs(self): file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( component_dir, "tests", "tags.yml" ) + file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( + component_dir, "tests", "main.nf.test" + ) return file_paths diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test new file mode 100644 index 0000000000..f38f69af49 --- /dev/null +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -0,0 +1,53 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core modules create-test-snap {{ component_name }} +nextflow_process { + + name "Test Process {{ component_name_underscore|upper }}" + script ../main.nf + process "{{ component_name_underscore|upper }}" + + tag 'modules' + tag 'modules_nfcore' + tag '{{ component_name }}' + {%- if subtool %} + tag '{{ component }}' + tag '{{ subtool }}' + {%- endif %} + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used + test("sarscov2 - bam") { + + // TODO nf-core: If you are created a test for a chained module + // (the module requires running more than one process to generate the required output) + // add the 'setup' method here. + // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules). + + when { + params { + outdir = "$outputDir"} + process { + """ + // define inputs of the process here. Example: + {% if has_meta %} + input = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + {%- else %} + input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + {%- endif %} + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + ) + } + + } + +} diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/modules/tests/tags.yml index 93f8228378..5d60d3a953 100644 --- a/nf_core/module-template/modules/tests/tags.yml +++ b/nf_core/module-template/modules/tests/tags.yml @@ -1,2 +1,2 @@ -{ { component_name_underscore } }: +{{ component_name_underscore }}: - "modules/{{ org }}/{{ component_dir }}/**" From 79215fb2eb3e55fa8d1542fbaeef4c87fd56e7fc Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 14:04:53 +0200 Subject: [PATCH 014/158] generalize nextflow_cmd to run_cmd to use the same structure for nf-test --- nf_core/list.py | 8 ++++---- nf_core/utils.py | 18 +++++++++++------- tests/test_download.py | 6 +++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/nf_core/list.py b/nf_core/list.py index 2bb90cb00e..c6c4bdf756 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -64,10 +64,10 @@ def get_local_wf(workflow, revision=None): # Wasn't local, fetch it log.info(f"Downloading workflow: {workflow} ({revision})") - pull_cmd = f"nextflow pull {workflow}" + pull_cmd = f"pull {workflow}" if revision is not None: pull_cmd += f" -r {revision}" - nf_core.utils.nextflow_cmd(pull_cmd) + nf_core.utils.run_cmd("nextflow", pull_cmd) local_wf = LocalWorkflow(workflow) local_wf.get_local_nf_workflow_details() return local_wf.local_path @@ -128,7 +128,7 @@ def get_local_nf_workflows(self): # Fetch details about local cached pipelines with `nextflow list` else: log.debug("Getting list of local nextflow workflows") - nflist_raw = nf_core.utils.nextflow_cmd("nextflow list") + nflist_raw = nf_core.utils.run_cmd("nextflow", "list") for wf_name in nflist_raw.splitlines(): if not str(wf_name).startswith("nf-core/"): self.local_unmatched.append(wf_name) @@ -342,7 +342,7 @@ def get_local_nf_workflow_details(self): # Use `nextflow info` to get more details about the workflow else: - nfinfo_raw = str(nf_core.utils.nextflow_cmd(f"nextflow info -d {self.full_name}")) + nfinfo_raw = str(nf_core.utils.run_cmd("nextflow", f"info -d {self.full_name}")) re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} for key, pattern in re_patterns.items(): m = re.search(pattern, nfinfo_raw) diff --git a/nf_core/utils.py b/nf_core/utils.py index 8290ed1780..00e16276dc 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -17,6 +17,7 @@ import sys import time from pathlib import Path +from typing import ByteString, Optional, Union import git import prompt_toolkit @@ -269,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = nextflow_cmd(f"nextflow config -flat {wf_path}") + nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}") for l in nfconfig_raw.splitlines(): ul = l.decode("utf-8") try: @@ -303,17 +304,20 @@ def fetch_wf_config(wf_path, cache_config=True): return config -def nextflow_cmd(cmd): - """Run a Nextflow command and capture the output. Handle errors nicely""" +def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]: + """Run a specified command and capture the output. Handle errors nicely.""" + full_cmd = f"{executable} {cmd}" try: - nf_proc = subprocess.run(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - return nf_proc.stdout + proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + return proc.stdout except OSError as e: if e.errno == errno.ENOENT: - raise AssertionError("It looks like Nextflow is not installed. It is required for most nf-core functions.") + raise AssertionError( + f"It looks like {executable} is not installed. Please ensure it is available in your PATH." + ) except subprocess.CalledProcessError as e: raise AssertionError( - f"Command '{cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" + f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" ) diff --git a/tests/test_download.py b/tests/test_download.py index ee0744f660..9416d462e4 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -16,7 +16,7 @@ import nf_core.utils from nf_core.download import ContainerError, DownloadWorkflow, WorkflowRepo from nf_core.synced_repo import SyncedRepo -from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, nextflow_cmd +from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR, run_cmd from .utils import with_temporary_file, with_temporary_folder @@ -156,8 +156,8 @@ def test_find_container_images_config_basic(self, tmp_path, mock_fetch_wf_config @mock.patch("nf_core.utils.fetch_wf_config") def test__find_container_images_config_nextflow(self, tmp_path, mock_fetch_wf_config): download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path) - nfconfig_raw = nextflow_cmd( - f"nextflow config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}" + nfconfig_raw = run_cmd( + "nextflow", f"config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}" ) config = {} for l in nfconfig_raw.splitlines(): From 7cab28667de5d72472494f9a5de25c0e04152b0f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 15:03:57 +0200 Subject: [PATCH 015/158] initial changes for nf-test --- nf_core/components/create.py | 102 ++++++++---------- nf_core/module-template/tests/main.nf | 18 ---- nf_core/module-template/tests/nextflow.config | 5 - nf_core/module-template/tests/test.yml | 18 ---- 4 files changed, 46 insertions(+), 97 deletions(-) delete mode 100644 nf_core/module-template/tests/main.nf delete mode 100644 nf_core/module-template/tests/nextflow.config delete mode 100644 nf_core/module-template/tests/test.yml diff --git a/nf_core/components/create.py b/nf_core/components/create.py index e626de4aaa..dbc7f90ff5 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -10,11 +10,12 @@ import os import re import subprocess +from pathlib import Path +from typing import Optional import jinja2 import questionary import rich -import yaml from packaging.version import parse as parse_version import nf_core @@ -27,16 +28,16 @@ class ComponentCreate(ComponentCommand): def __init__( self, - component_type, - directory=".", - component="", - author=None, - process_label=None, - has_meta=None, - force=False, - conda_name=None, - conda_version=None, - empty_template=False, + component_type: str, + directory: str = ".", + component: str = "", + author: Optional[str] = None, + process_label: Optional[str] = None, + has_meta: Optional[str] = None, + force: bool = False, + conda_name: Optional[str] = None, + conda_version: Optional[str] = None, + empty_template: bool = False, ): super().__init__(component_type, directory) self.directory = directory @@ -71,23 +72,23 @@ def create(self): If is a pipeline, this function creates a file called: '/modules/local/tool.nf' - OR + OR '/modules/local/tool_subtool.nf' - OR for subworkflows + OR for subworkflows '/subworkflows/local/subworkflow_name.nf' If is a clone of nf-core/modules, it creates or modifies the following files: For modules: + ```tree modules/modules/nf-core/tool/subtool/ - * main.nf - * meta.yml - modules/tests/modules/nf-core/tool/subtool/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml + ├── main.nf + ├── meta.yml + └── tests + ├── main.nf.test + └── tags.yml + ``` The function will attempt to automatically find a Bioconda package called and matching Docker / Singularity images from BioContainers. @@ -147,32 +148,23 @@ def create(self): # Create component template with jinja2 self._render_template() - if self.repo_type == "modules": - # Add entry to pytest_modules.yml - try: - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "r") as fh: - pytest_modules_yml = yaml.safe_load(fh) - if self.subtool: - pytest_modules_yml[self.component_name] = [ - f"modules/{self.org}/{self.component}/{self.subtool}/**", - f"tests/modules/{self.org}/{self.component}/{self.subtool}/**", - ] + # generate nf-tests + nf_core.utils.run_cmd("nf-test", f"generate process {self.file_paths['main.nf']}") + # move generated main.nf.test file into test directory + + Path("main.nf.test").rename(self.file_paths[os.path.join(self.component_type, "test", "main.nf.test")]) + + # inside main.nf.test replace the path to the main.nf file with the relative path for the script + with open(self.file_paths["tests/main.nf.test"], "r") as f: + lines = f.readlines() + with open(self.file_paths["tests/main.nf.test"], "w") as f: + for line in lines: + if line.startswith("script"): + f.write(f"script ../main.nf\n") else: - pytest_modules_yml[ - ("" if self.component_type == "modules" else self.component_type + "/") + self.component_name - ] = [ - f"{self.component_type}/{self.org}/{self.component}/**", - f"tests/{self.component_type}/{self.org}/{self.component}/**", - ] - pytest_modules_yml = dict(sorted(pytest_modules_yml.items())) - with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh: - yaml.dump(pytest_modules_yml, fh, sort_keys=True, Dumper=nf_core.utils.custom_yaml_dumper()) - except FileNotFoundError: - raise UserWarning("Could not open 'tests/config/pytest_modules.yml' file!") + f.write(line) new_files = list(self.file_paths.values()) - if self.repo_type == "modules": - new_files.append(os.path.join(self.directory, "tests", "config", "pytest_modules.yml")) log.info("Created / edited following files:\n " + "\n ".join(new_files)) def _get_bioconda_tool(self): @@ -362,17 +354,12 @@ def _get_component_dirs(self): file_paths[os.path.join(self.component_type, "main.nf")] = component_file if self.repo_type == "modules": - software_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) - test_dir = os.path.join(self.directory, "tests", self.component_type, self.org, self.component_dir) + component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(software_dir) and not self.force_overwrite: + if os.path.exists(component_dir) and not self.force_overwrite: raise UserWarning( - f"{self.component_type[:-1]} directory exists: '{software_dir}'. Use '--force' to overwrite" - ) - if os.path.exists(test_dir) and not self.force_overwrite: - raise UserWarning( - f"{self.component_type[:-1]} test directory exists: '{test_dir}'. Use '--force' to overwrite" + f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) if self.component_type == "modules": @@ -403,11 +390,14 @@ def _get_component_dirs(self): # Set file paths # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(software_dir, "main.nf") - file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(software_dir, "meta.yml") - file_paths[os.path.join("tests", "main.nf")] = os.path.join(test_dir, "main.nf") - file_paths[os.path.join("tests", "test.yml")] = os.path.join(test_dir, "test.yml") - file_paths[os.path.join("tests", "nextflow.config")] = os.path.join(test_dir, "nextflow.config") + file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") + file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") + file_paths[os.path.join(self.component_type, "test", "main.nf.test")] = os.path.join( + component_dir, "test", "main.nf.test" + ) + file_paths[os.path.join(self.component_type, "test", "tags.yml")] = os.path.join( + component_dir, "test", "tags.yml" + ) return file_paths diff --git a/nf_core/module-template/tests/main.nf b/nf_core/module-template/tests/main.nf deleted file mode 100644 index fcb7195fe4..0000000000 --- a/nf_core/module-template/tests/main.nf +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { {{ component_name_underscore|upper }} } from '../../../../{{ "../" if subtool else "" }}modules/{{ org }}/{{ component_dir }}/main.nf' - -workflow test_{{ component_name_underscore }} { - {% if has_meta %} - input = [ - [ id:'test', single_end:false ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - {%- else %} - input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) - {%- endif %} - - {{ component_name_underscore|upper }} ( input ) -} diff --git a/nf_core/module-template/tests/nextflow.config b/nf_core/module-template/tests/nextflow.config deleted file mode 100644 index 50f50a7a35..0000000000 --- a/nf_core/module-template/tests/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} \ No newline at end of file diff --git a/nf_core/module-template/tests/test.yml b/nf_core/module-template/tests/test.yml deleted file mode 100644 index a2cedb73d3..0000000000 --- a/nf_core/module-template/tests/test.yml +++ /dev/null @@ -1,18 +0,0 @@ -{%- if not_empty_template -%} -## TODO nf-core: Please run the following command to build this file: -# nf-core modules create-test-yml {{ tool }}{%- if subtool %}/{{ subtool }}{%- endif %} -{% endif -%} -- name: "{{ component }}{{ ' '+subtool if subtool else '' }}" - command: nextflow run ./tests/modules/{{ org }}/{{ component_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config - tags: - - "{{ component }}{% if subtool -%}" - - "{{ component }}/{{ subtool }}{%- endif %}" - files: - {% if not_empty_template -%} - - path: "output/{{ component }}/test.bam" - md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: "output/{{ component }}/versions.yml" - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b - {% else -%} - - path: "" - {%- endif %} From f41d410028051270a9a1d54a18d7a0581bb859fe Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 15:24:10 +0200 Subject: [PATCH 016/158] move tags.yml to correct position --- nf_core/module-template/modules/tests/tags.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 nf_core/module-template/modules/tests/tags.yml diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/modules/tests/tags.yml new file mode 100644 index 0000000000..93f8228378 --- /dev/null +++ b/nf_core/module-template/modules/tests/tags.yml @@ -0,0 +1,2 @@ +{ { component_name_underscore } }: + - "modules/{{ org }}/{{ component_dir }}/**" From cbc398a90efdeed47a3d152185996b532f231b49 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 16:23:53 +0200 Subject: [PATCH 017/158] convert all cmds --- nf_core/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 00e16276dc..6d2ca24a07 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -270,7 +270,7 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = nextflow_cmd("nextflow", f"config -flat {wf_path}") + nfconfig_raw = run_cmd("nextflow", f"config -flat {wf_path}") for l in nfconfig_raw.splitlines(): ul = l.decode("utf-8") try: @@ -307,6 +307,7 @@ def fetch_wf_config(wf_path, cache_config=True): def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]: """Run a specified command and capture the output. Handle errors nicely.""" full_cmd = f"{executable} {cmd}" + log.debug(f"Running command: {full_cmd}") try: proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) return proc.stdout From 8d7794a064d67d11d04d257dc1f2fe5905d7b7f4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 16:32:19 +0200 Subject: [PATCH 018/158] first running version --- nf_core/components/create.py | 59 +++++++++++++++--------- nf_core/module-template/modules/meta.yml | 4 +- 2 files changed, 40 insertions(+), 23 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index dbc7f90ff5..fe08391822 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -147,23 +147,45 @@ def create(self): # Create component template with jinja2 self._render_template() + log.info(f"Created component template: '{self.component_name}'") # generate nf-tests - nf_core.utils.run_cmd("nf-test", f"generate process {self.file_paths['main.nf']}") - # move generated main.nf.test file into test directory + nf_core.utils.run_cmd( + "nf-test", f"generate process {self.file_paths[os.path.join(self.component_type, 'main.nf')]}" + ) + + # move generated main.nf.test file into tests directory + tests_dir = Path(self.directory, self.component_type, self.org, self.component_dir, "tests") + tests_dir.mkdir(exist_ok=True) + + Path(tests_dir.parent, "main.nf.test").rename(tests_dir / "main.nf.test") - Path("main.nf.test").rename(self.file_paths[os.path.join(self.component_type, "test", "main.nf.test")]) + self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( + self.directory, self.component_type, self.org, self.component_dir, "tests", "main.nf.test" + ) + log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") # inside main.nf.test replace the path to the main.nf file with the relative path for the script - with open(self.file_paths["tests/main.nf.test"], "r") as f: + with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: lines = f.readlines() - with open(self.file_paths["tests/main.nf.test"], "w") as f: + with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: for line in lines: - if line.startswith("script"): - f.write(f"script ../main.nf\n") + if line.strip().startswith("script "): + log.debug(f"Replacing script path in nf-test: '{line.strip()}'") + f.write(f" script ../main.nf\n") + # add tag elements after process line + if line.strip().startswith("process "): + log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") + f.write(line) + f.write("\n") + f.write(f" tag 'modules'\n") + f.write(f" tag 'modules_nfcore'\n") + f.write(f" tag '{self.component_name_underscore}'\n") + f.write(f" tag '{self.component}'\n") + if self.subtool: + f.write(f" tag '{self.subtool}'\n") else: f.write(line) - new_files = list(self.file_paths.values()) log.info("Created / edited following files:\n " + "\n ".join(new_files)) @@ -265,9 +287,14 @@ def _render_template(self): log.debug(f"Rendering template file: '{template_fn}'") j_template = env.get_template(template_fn) object_attrs["nf_core_version"] = nf_core.__version__ - rendered_output = j_template.render(object_attrs) + try: + rendered_output = j_template.render(object_attrs) + except Exception as e: + log.error(f"Could not render template file '{template_fn}':\n{e}") + raise e # Write output to the target file + log.debug(f"Writing output to: '{dest_fn}'") os.makedirs(os.path.dirname(dest_fn), exist_ok=True) with open(dest_fn, "w") as fh: log.debug(f"Writing output to: '{dest_fn}'") @@ -367,17 +394,10 @@ def _get_component_dirs(self): parent_tool_main_nf = os.path.join( self.directory, self.component_type, self.org, self.component, "main.nf" ) - parent_tool_test_nf = os.path.join( - self.directory, self.component_type, self.org, self.component, "main.nf" - ) if self.subtool and os.path.exists(parent_tool_main_nf): raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) - if self.subtool and os.path.exists(parent_tool_test_nf): - raise UserWarning( - f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{self.component_name}'" - ) # If no subtool, check that there isn't already a tool/subtool tool_glob = glob.glob( @@ -392,11 +412,8 @@ def _get_component_dirs(self): # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") - file_paths[os.path.join(self.component_type, "test", "main.nf.test")] = os.path.join( - component_dir, "test", "main.nf.test" - ) - file_paths[os.path.join(self.component_type, "test", "tags.yml")] = os.path.join( - component_dir, "test", "tags.yml" + file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( + component_dir, "tests", "tags.yml" ) return file_paths diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index aea3c36aa3..c5d9c042b4 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -30,7 +30,7 @@ input: type: map description: | Groovy Map containing sample information - e.g. `[ id:'test', single_end:false ]` + e.g. `[ id:'sample1', single_end:false ]` {% endif %} {% if not_empty_template -%} ## TODO nf-core: Delete / customise this example input @@ -49,7 +49,7 @@ output: type: map description: | Groovy Map containing sample information - e.g. `[ id:'test', single_end:false ]` + e.g. `[ id:'sample1', single_end:false ]` {% endif %} - versions: type: file From 1a58690aada02b227f0a31d0ba0b3e8ee655f660 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 17:24:49 +0200 Subject: [PATCH 019/158] replace params --- nf_core/components/create.py | 53 ++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index fe08391822..b1c94566d8 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -11,7 +11,7 @@ import re import subprocess from pathlib import Path -from typing import Optional +from typing import Dict, Optional import jinja2 import questionary @@ -57,7 +57,7 @@ def __init__( self.bioconda = None self.singularity_container = None self.docker_container = None - self.file_paths = {} + self.file_paths: Dict[str, str] = {} self.not_empty_template = not empty_template def create(self): @@ -165,27 +165,40 @@ def create(self): ) log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") - # inside main.nf.test replace the path to the main.nf file with the relative path for the script + # Read the content from the file with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: lines = f.readlines() + + # Construct the modified content + modified_content = [] + for line in lines: + if line.strip().startswith("script "): + log.debug(f"Replacing script path in nf-test: '{line.strip()}'") + modified_content.append(f" script ../main.nf\n") + elif line.strip().startswith('process "'): + log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") + modified_content.append(line) + modified_content.append("\n") + modified_content.append(f" tag 'modules'\n") + modified_content.append(f" tag 'modules_nfcore'\n") + modified_content.append(f" tag '{self.component_name_underscore}'\n") + modified_content.append(f" tag '{self.component}'\n") + if self.subtool: + modified_content.append(f" tag '{self.subtool}'\n") + else: + modified_content.append(line) + + # Combine the modified content into one string + content = "".join(modified_content) + + # Apply the regex substitution + replacement = ' outdir = "$outputDir"' + content = re.sub(r"(params\s*{)\s*[^}]+(})", r"\1\n " + replacement + r"\2", content, flags=re.DOTALL) + + # Write the final content back to the file with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: - for line in lines: - if line.strip().startswith("script "): - log.debug(f"Replacing script path in nf-test: '{line.strip()}'") - f.write(f" script ../main.nf\n") - # add tag elements after process line - if line.strip().startswith("process "): - log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") - f.write(line) - f.write("\n") - f.write(f" tag 'modules'\n") - f.write(f" tag 'modules_nfcore'\n") - f.write(f" tag '{self.component_name_underscore}'\n") - f.write(f" tag '{self.component}'\n") - if self.subtool: - f.write(f" tag '{self.subtool}'\n") - else: - f.write(line) + f.write(content) + new_files = list(self.file_paths.values()) log.info("Created / edited following files:\n " + "\n ".join(new_files)) From 86db72b77c699d278438997eefc12cb1ed0f06d8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 24 Oct 2023 17:25:05 +0200 Subject: [PATCH 020/158] fix typing errors --- nf_core/utils.py | 5 +++-- requirements-dev.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 6d2ca24a07..955ef524bc 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -54,10 +54,10 @@ ) NFCORE_CACHE_DIR = os.path.join( - os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME"), ".cache")), + os.environ.get("XDG_CACHE_HOME", os.path.join(os.getenv("HOME") or "", ".cache")), "nfcore", ) -NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME"), ".config")), "nfcore") +NFCORE_DIR = os.path.join(os.environ.get("XDG_CONFIG_HOME", os.path.join(os.getenv("HOME") or "", ".config")), "nfcore") def fetch_remote_version(source_url): @@ -329,6 +329,7 @@ def setup_nfcore_dir(): """ if not os.path.exists(NFCORE_DIR): os.makedirs(NFCORE_DIR) + return True def setup_requests_cachedir(): diff --git a/requirements-dev.txt b/requirements-dev.txt index 0f650b9e5c..c94874b193 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,3 +8,4 @@ Sphinx sphinx-rtd-theme mypy types-PyYAML +types-requests From 7a7fd93adf0c247582946ccf8ad715f693cdbe67 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Oct 2023 17:15:25 +0200 Subject: [PATCH 021/158] add TODO comments to main.nf.test and fix tags.yml template rendering --- nf_core/components/create.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index b1c94566d8..9d5f5e270d 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -172,19 +172,44 @@ def create(self): # Construct the modified content modified_content = [] for line in lines: - if line.strip().startswith("script "): + if line.strip().startswith("nextflow_process"): + log.debug("Adding TODO nf-core comment to nf-test") + modified_content.append( + "// TODO nf-core: Once you have added the required tests, please run the following command to build this file:\n" + ) + modified_content.append(f"// nf-core {self.component_type} create-test-snap {self.component_name}\n") + modified_content.append(line) + elif line.strip().startswith("script "): log.debug(f"Replacing script path in nf-test: '{line.strip()}'") modified_content.append(f" script ../main.nf\n") elif line.strip().startswith('process "'): log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") modified_content.append(line) modified_content.append("\n") - modified_content.append(f" tag 'modules'\n") - modified_content.append(f" tag 'modules_nfcore'\n") - modified_content.append(f" tag '{self.component_name_underscore}'\n") - modified_content.append(f" tag '{self.component}'\n") + modified_content.append(" tag 'modules'\n") + modified_content.append(" tag 'modules_nfcore'\n") + modified_content.append(f" tag '{self.component_name}'\n") if self.subtool: + modified_content.append(f" tag '{self.component}'\n") modified_content.append(f" tag '{self.subtool}'\n") + elif line.strip().startswith('test("Should run without failures")'): + log.debug("Adding TODO nf-core comment to change test name") + modified_content.append( + " //TODO nf-core: Change the test name preferably indicating the test-data and file-format used\n" + ) + modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') + modified_content.append(line) + elif line.strip().startswith("assert process.success"): + log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") + modified_content.append(" assertAll(\n") + modified_content.append(" { assert process.success },\n") + elif line.strip().startswith("assert snapshot(process.out).match()"): + log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") + modified_content.append(" { assert snapshot(process.out).match() }\n") + modified_content.append( + " //TODO nf-core: Add all required assertions to verify the test output.\n" + ) + modified_content.append(" )\n") else: modified_content.append(line) From 11773ef7939e45a2e0d308520c8143a87b3fbd45 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 25 Oct 2023 17:24:14 +0200 Subject: [PATCH 022/158] add todo comment for chained modules --- nf_core/components/create.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 9d5f5e270d..4e8bd5f1d4 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -199,6 +199,13 @@ def create(self): ) modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') modified_content.append(line) + log.debug("Adding TODO nf-core comment about using a setup method") + modified_content.append( + "\n //TODO nf-core: If you are created a test for a chained module\n" + " //(the module requires running more than one process to generate the required output)\n" + " //add the 'setup' method here.\n" + " //You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" + ) elif line.strip().startswith("assert process.success"): log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") modified_content.append(" assertAll(\n") From eafb3b5f47c409f6a6d81c6f476072572c285c6e Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:04:20 +0200 Subject: [PATCH 023/158] add real example to main.nf.test --- nf_core/components/create.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 4e8bd5f1d4..e55daa562c 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -195,17 +195,30 @@ def create(self): elif line.strip().startswith('test("Should run without failures")'): log.debug("Adding TODO nf-core comment to change test name") modified_content.append( - " //TODO nf-core: Change the test name preferably indicating the test-data and file-format used\n" + " // TODO nf-core: Change the test name preferably indicating the test-data and file-format used. Example:\n" ) - modified_content.append(' //Example: test("homo_sapiens - [bam, bai, bed] - fasta - fai")\n') - modified_content.append(line) + modified_content.append(' test("sarscov2 - bam") {\n') log.debug("Adding TODO nf-core comment about using a setup method") modified_content.append( - "\n //TODO nf-core: If you are created a test for a chained module\n" - " //(the module requires running more than one process to generate the required output)\n" - " //add the 'setup' method here.\n" - " //You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" + "\n // TODO nf-core: If you are created a test for a chained module\n" + " // (the module requires running more than one process to generate the required output)\n" + " // add the 'setup' method here.\n" + " // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" ) + elif line.strip().startswith("// input[0]"): + log.debug(f"Replacing input data example: '{line.strip()}'") + if self.has_meta: + modified_content.append( + " input = [\n" + " [ id:'test', single_end:false ], // meta map\n" + " file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)\n" + " ]\n" + ) + else: + modified_content.append( + " // input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)\n" + ) + elif line.strip().startswith("assert process.success"): log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") modified_content.append(" assertAll(\n") From 56086c61bc6f4bf7ee276921085ce3fa1cae12a6 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:16:59 +0200 Subject: [PATCH 024/158] add tags.yml to prettierignore --- .prettierignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prettierignore b/.prettierignore index 4cd77bb4ed..1ecc7d3ada 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,4 @@ slackreport.json docs/api/_build testing nf_core/module-template/modules/meta.yml -nf_core/module-template/tests/test.yml +nf-core/module-template/modules/tests/tags.yml From 63c31884b02ddf0bd2d10c11cb016f8dc22577d4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 26 Oct 2023 12:24:02 +0200 Subject: [PATCH 025/158] add main.nf.test to template instead of using nf-test --- .prettierignore | 2 +- nf_core/components/create.py | 100 +----------------- .../modules/tests/main.nf.test | 53 ++++++++++ .../module-template/modules/tests/tags.yml | 2 +- 4 files changed, 59 insertions(+), 98 deletions(-) create mode 100644 nf_core/module-template/modules/tests/main.nf.test diff --git a/.prettierignore b/.prettierignore index 1ecc7d3ada..9f16831c4a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -4,4 +4,4 @@ slackreport.json docs/api/_build testing nf_core/module-template/modules/meta.yml -nf-core/module-template/modules/tests/tags.yml +nf_core/module-template/modules/tests/tags.yml diff --git a/nf_core/components/create.py b/nf_core/components/create.py index e55daa562c..8b96a02076 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -149,103 +149,8 @@ def create(self): self._render_template() log.info(f"Created component template: '{self.component_name}'") - # generate nf-tests - nf_core.utils.run_cmd( - "nf-test", f"generate process {self.file_paths[os.path.join(self.component_type, 'main.nf')]}" - ) - - # move generated main.nf.test file into tests directory - tests_dir = Path(self.directory, self.component_type, self.org, self.component_dir, "tests") - tests_dir.mkdir(exist_ok=True) - - Path(tests_dir.parent, "main.nf.test").rename(tests_dir / "main.nf.test") - - self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( - self.directory, self.component_type, self.org, self.component_dir, "tests", "main.nf.test" - ) - - log.debug(f"Created nf-test: '{self.file_paths[os.path.join(self.component_type, 'tests', 'main.nf.test')]}'") - # Read the content from the file - with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "r") as f: - lines = f.readlines() - - # Construct the modified content - modified_content = [] - for line in lines: - if line.strip().startswith("nextflow_process"): - log.debug("Adding TODO nf-core comment to nf-test") - modified_content.append( - "// TODO nf-core: Once you have added the required tests, please run the following command to build this file:\n" - ) - modified_content.append(f"// nf-core {self.component_type} create-test-snap {self.component_name}\n") - modified_content.append(line) - elif line.strip().startswith("script "): - log.debug(f"Replacing script path in nf-test: '{line.strip()}'") - modified_content.append(f" script ../main.nf\n") - elif line.strip().startswith('process "'): - log.debug(f"Adding tag elements to nf-test: '{line.strip()}'") - modified_content.append(line) - modified_content.append("\n") - modified_content.append(" tag 'modules'\n") - modified_content.append(" tag 'modules_nfcore'\n") - modified_content.append(f" tag '{self.component_name}'\n") - if self.subtool: - modified_content.append(f" tag '{self.component}'\n") - modified_content.append(f" tag '{self.subtool}'\n") - elif line.strip().startswith('test("Should run without failures")'): - log.debug("Adding TODO nf-core comment to change test name") - modified_content.append( - " // TODO nf-core: Change the test name preferably indicating the test-data and file-format used. Example:\n" - ) - modified_content.append(' test("sarscov2 - bam") {\n') - log.debug("Adding TODO nf-core comment about using a setup method") - modified_content.append( - "\n // TODO nf-core: If you are created a test for a chained module\n" - " // (the module requires running more than one process to generate the required output)\n" - " // add the 'setup' method here.\n" - " // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules).\n" - ) - elif line.strip().startswith("// input[0]"): - log.debug(f"Replacing input data example: '{line.strip()}'") - if self.has_meta: - modified_content.append( - " input = [\n" - " [ id:'test', single_end:false ], // meta map\n" - " file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)\n" - " ]\n" - ) - else: - modified_content.append( - " // input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)\n" - ) - - elif line.strip().startswith("assert process.success"): - log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") - modified_content.append(" assertAll(\n") - modified_content.append(" { assert process.success },\n") - elif line.strip().startswith("assert snapshot(process.out).match()"): - log.debug(f"Replacing assertion lines in nf-test: '{line.strip()}'") - modified_content.append(" { assert snapshot(process.out).match() }\n") - modified_content.append( - " //TODO nf-core: Add all required assertions to verify the test output.\n" - ) - modified_content.append(" )\n") - else: - modified_content.append(line) - - # Combine the modified content into one string - content = "".join(modified_content) - - # Apply the regex substitution - replacement = ' outdir = "$outputDir"' - content = re.sub(r"(params\s*{)\s*[^}]+(})", r"\1\n " + replacement + r"\2", content, flags=re.DOTALL) - - # Write the final content back to the file - with open(self.file_paths[os.path.join(self.component_type, "tests", "main.nf.test")], "w") as f: - f.write(content) - new_files = list(self.file_paths.values()) - log.info("Created / edited following files:\n " + "\n ".join(new_files)) + log.info("Created following files:\n " + "\n ".join(new_files)) def _get_bioconda_tool(self): """ @@ -473,6 +378,9 @@ def _get_component_dirs(self): file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( component_dir, "tests", "tags.yml" ) + file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( + component_dir, "tests", "main.nf.test" + ) return file_paths diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test new file mode 100644 index 0000000000..f38f69af49 --- /dev/null +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -0,0 +1,53 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core modules create-test-snap {{ component_name }} +nextflow_process { + + name "Test Process {{ component_name_underscore|upper }}" + script ../main.nf + process "{{ component_name_underscore|upper }}" + + tag 'modules' + tag 'modules_nfcore' + tag '{{ component_name }}' + {%- if subtool %} + tag '{{ component }}' + tag '{{ subtool }}' + {%- endif %} + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used + test("sarscov2 - bam") { + + // TODO nf-core: If you are created a test for a chained module + // (the module requires running more than one process to generate the required output) + // add the 'setup' method here. + // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules). + + when { + params { + outdir = "$outputDir"} + process { + """ + // define inputs of the process here. Example: + {% if has_meta %} + input = [ + [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) + ] + {%- else %} + input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + {%- endif %} + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + //TODO nf-core: Add all required assertions to verify the test output. + ) + } + + } + +} diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/modules/tests/tags.yml index 93f8228378..5d60d3a953 100644 --- a/nf_core/module-template/modules/tests/tags.yml +++ b/nf_core/module-template/modules/tests/tags.yml @@ -1,2 +1,2 @@ -{ { component_name_underscore } }: +{{ component_name_underscore }}: - "modules/{{ org }}/{{ component_dir }}/**" From 40381315b30258de632df1f856819f95405c023a Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 31 Oct 2023 11:03:04 +0100 Subject: [PATCH 026/158] modules create tests check for main.nf.test --- tests/modules/create.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/modules/create.py b/tests/modules/create.py index 98e498c1b0..7cfba484a1 100644 --- a/tests/modules/create.py +++ b/tests/modules/create.py @@ -48,7 +48,7 @@ def test_modules_create_nfcore_modules(self): with requests_cache.disabled(): module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "main.nf")) - assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "fastqc", "main.nf")) + assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "fastqc", "tests", "main.nf.test")) def test_modules_create_nfcore_modules_subtool(self): @@ -62,4 +62,6 @@ def test_modules_create_nfcore_modules_subtool(self): with requests_cache.disabled(): module_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "main.nf")) - assert os.path.exists(os.path.join(self.nfcore_modules, "tests", "modules", "nf-core", "star", "index", "main.nf")) + assert os.path.exists( + os.path.join(self.nfcore_modules, "modules", "nf-core", "star", "index", "tests", "main.nf.test") + ) From 43bf2180b39df6d05e706610275be806edb38448 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 31 Oct 2023 11:04:38 +0100 Subject: [PATCH 027/158] remove outputDir from the modules template --- nf_core/module-template/modules/tests/main.nf.test | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index f38f69af49..c1df109ab4 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -23,8 +23,6 @@ nextflow_process { // You can find more information about how to use a 'setup' method in the docs (https://nf-co.re/docs/contributing/modules#steps-for-creating-nf-test-for-chained-modules). when { - params { - outdir = "$outputDir"} process { """ // define inputs of the process here. Example: From 4ffc3cf2cbc4902d9a85422d188dc1b0c8328909 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 31 Oct 2023 12:27:47 +0100 Subject: [PATCH 028/158] add nf-test to subworkflows --- .prettierignore | 1 + nf_core/components/create.py | 19 ++++----- .../modules/tests/main.nf.test | 4 +- .../subworkflows/tests/main.nf.test | 40 +++++++++++++++++++ .../subworkflows/tests/tags.yml | 2 + nf_core/subworkflow-template/tests/main.nf | 18 --------- .../tests/nextflow.config | 5 --- nf_core/subworkflow-template/tests/test.yml | 12 ------ 8 files changed, 55 insertions(+), 46 deletions(-) create mode 100644 nf_core/subworkflow-template/subworkflows/tests/main.nf.test create mode 100644 nf_core/subworkflow-template/subworkflows/tests/tags.yml delete mode 100644 nf_core/subworkflow-template/tests/main.nf delete mode 100644 nf_core/subworkflow-template/tests/nextflow.config delete mode 100644 nf_core/subworkflow-template/tests/test.yml diff --git a/.prettierignore b/.prettierignore index 9f16831c4a..778fb4fb2c 100644 --- a/.prettierignore +++ b/.prettierignore @@ -5,3 +5,4 @@ docs/api/_build testing nf_core/module-template/modules/meta.yml nf_core/module-template/modules/tests/tags.yml +nf_core/subworkflow-template/subworkflows/tests/tags.yml diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 8b96a02076..64d86be2e8 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -82,7 +82,7 @@ def create(self): For modules: ```tree - modules/modules/nf-core/tool/subtool/ + modules/nf-core/tool/subtool/ ├── main.nf ├── meta.yml └── tests @@ -94,14 +94,15 @@ def create(self): and matching Docker / Singularity images from BioContainers. For subworkflows: - subworkflows/nf-core/subworkflow_name/ - * main.nf - * meta.yml - tests/subworkflows/nf-core/subworkflow_name/ - * main.nf - * test.yml - * nextflow.config - tests/config/pytest_modules.yml + + ```tree + subworkflows/nf-core/tool/subtool/ + ├── main.nf + ├── meta.yml + └── tests + ├── main.nf.test + └── tags.yml + ``` """ diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index c1df109ab4..ccfb18d96a 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -1,5 +1,5 @@ // TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core modules create-test-snap {{ component_name }} +// nf-core modules create-nf-test {{ component_name }} nextflow_process { name "Test Process {{ component_name_underscore|upper }}" @@ -25,7 +25,7 @@ nextflow_process { when { process { """ - // define inputs of the process here. Example: + // TODO nf-core: define inputs of the process here. Example: {% if has_meta %} input = [ [ id:'test', single_end:false ], // meta map diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test new file mode 100644 index 0000000000..826e575bd5 --- /dev/null +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -0,0 +1,40 @@ +// TODO nf-core: Once you have added the required tests, please run the following command to build this file: +// nf-core subworkflows create-nf-test {{ component_name }} +nextflow_workflow { + + name "Test Workflow {{ component_name_underscore|upper }}" + script "../main.nf" + workflow "{{ component_name_underscore|upper }}" + + tag 'subworkflows' + tag 'subworkflows_nfcore' + tag '{{ component_name }}' + // TODO nf-core: Add tags for all modules used within this subworkflow + + + // TODO nf-core: Change the test name preferably indicating the test-data and file-format used + test("sarscov2 - bam - single_end") { + + when { + workflow { + """ + // TODO nf-core: define inputs of the workflow here. Example: + input[0] = [ [ id:'test', single_end:false ], // meta map + file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) + ] + input[1] = [ [ id:'genome' ], + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert workflow.success}, + { assert snapshot(workflow.out).match()} + //TODO nf-core: Add all required assertions to verify the test output. + ) + } + } +} diff --git a/nf_core/subworkflow-template/subworkflows/tests/tags.yml b/nf_core/subworkflow-template/subworkflows/tests/tags.yml new file mode 100644 index 0000000000..8cde627a13 --- /dev/null +++ b/nf_core/subworkflow-template/subworkflows/tests/tags.yml @@ -0,0 +1,2 @@ +{{ component_name_underscore }}: + - subworkflows/{{ org }}/{{ component_dir }}/** diff --git a/nf_core/subworkflow-template/tests/main.nf b/nf_core/subworkflow-template/tests/main.nf deleted file mode 100644 index e09cc50af8..0000000000 --- a/nf_core/subworkflow-template/tests/main.nf +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env nextflow - -nextflow.enable.dsl = 2 - -include { {{ component_name_underscore|upper }} } from '../../../../subworkflows/{{ org }}/{{ subworkflow_dir }}/main.nf' - -workflow test_{{ component_name_underscore }} { - {% if has_meta %} - input = [ - [ id:'test' ], // meta map - file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true) - ] - {%- else %} - input = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true) - {%- endif %} - - {{ component_name_underscore|upper }} ( input ) -} diff --git a/nf_core/subworkflow-template/tests/nextflow.config b/nf_core/subworkflow-template/tests/nextflow.config deleted file mode 100644 index 8730f1c4b9..0000000000 --- a/nf_core/subworkflow-template/tests/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } - -} diff --git a/nf_core/subworkflow-template/tests/test.yml b/nf_core/subworkflow-template/tests/test.yml deleted file mode 100644 index 7bca1616f3..0000000000 --- a/nf_core/subworkflow-template/tests/test.yml +++ /dev/null @@ -1,12 +0,0 @@ -## TODO nf-core: Please run the following command to build this file: -# nf-core subworkflows create-test-yml {{ component_name_underscore }} -- name: "{{ component_name_underscore }}" - command: nextflow run ./tests/subworkflows/{{ org }}/{{ subworkflow_dir }} -entry test_{{ component_name_underscore }} -c ./tests/config/nextflow.config - tags: - - "subworkflows" - - "subworkflows/{{ component_name_underscore }}" - files: - - path: "output/{{ component_name_underscore }}/test.bam" - md5sum: e667c7caad0bc4b7ac383fd023c654fc - - path: output/{{ component_name_underscore }}/versions.yml - md5sum: a01fe51bc4c6a3a6226fbf77b2c7cf3b From db1025339410768ff371455ec8068bccb18dec43 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 31 Oct 2023 12:46:50 +0100 Subject: [PATCH 029/158] subworkflows create tests check for main.nf.test --- tests/subworkflows/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/subworkflows/create.py b/tests/subworkflows/create.py index 60ee6add9a..94c2a66331 100644 --- a/tests/subworkflows/create.py +++ b/tests/subworkflows/create.py @@ -33,5 +33,5 @@ def test_subworkflows_create_nfcore_modules(self): subworkflow_create.create() assert os.path.exists(os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf")) assert os.path.exists( - os.path.join(self.nfcore_modules, "tests", "subworkflows", "nf-core", "test_subworkflow", "main.nf") + os.path.join(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test") ) From 80ddddbb3ca7628468caf1ae3d6ab9fa3d58c9f1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 1 Nov 2023 09:46:50 +0100 Subject: [PATCH 030/158] fix tags for subtools, add version assertion to template --- nf_core/module-template/modules/tests/main.nf.test | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index ccfb18d96a..3f65b97601 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -8,11 +8,10 @@ nextflow_process { tag 'modules' tag 'modules_nfcore' - tag '{{ component_name }}' {%- if subtool %} tag '{{ component }}' - tag '{{ subtool }}' {%- endif %} + tag '{{ component_name }}' // TODO nf-core: Change the test name preferably indicating the test-data and file-format used test("sarscov2 - bam") { @@ -41,7 +40,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out).match() } + { assert snapshot(process.out.versions).match("versions") } //TODO nf-core: Add all required assertions to verify the test output. ) } From 617010e320c55d61e907ed94feac94710f537060 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 1 Nov 2023 10:52:13 +0100 Subject: [PATCH 031/158] fix quotes in templates --- nf_core/module-template/modules/tests/main.nf.test | 10 +++++----- .../subworkflows/tests/main.nf.test | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index 3f65b97601..0e08f89e87 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -3,15 +3,15 @@ nextflow_process { name "Test Process {{ component_name_underscore|upper }}" - script ../main.nf + script "../main.nf" process "{{ component_name_underscore|upper }}" - tag 'modules' - tag 'modules_nfcore' + tag "modules" + tag "modules_nfcore" {%- if subtool %} - tag '{{ component }}' + tag "{{ component }}" {%- endif %} - tag '{{ component_name }}' + tag "{{ component_name }}" // TODO nf-core: Change the test name preferably indicating the test-data and file-format used test("sarscov2 - bam") { diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index 826e575bd5..b95cfeb383 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -6,9 +6,9 @@ nextflow_workflow { script "../main.nf" workflow "{{ component_name_underscore|upper }}" - tag 'subworkflows' - tag 'subworkflows_nfcore' - tag '{{ component_name }}' + tag "subworkflows" + tag "subworkflows_nfcore" + tag "{{ component_name }}" // TODO nf-core: Add tags for all modules used within this subworkflow From 04efb402786257db7ffef097ee088c3828fa4bb6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 1 Nov 2023 16:48:21 +0100 Subject: [PATCH 032/158] add first version of the `create-snapshot` command --- nf_core/__main__.py | 26 ++++++++++++-------------- nf_core/list.py | 28 ++++++++++++++++------------ nf_core/utils.py | 36 +++++++++++++++++++++--------------- 3 files changed, 49 insertions(+), 41 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 72762ff026..dc1f5c4e07 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -849,34 +849,32 @@ def create_module( sys.exit(1) -# nf-core modules create-test-yml -@modules.command("create-test-yml") +# nf-core modules create-snapshot +@modules.command("create-snapshot") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") -@click.option("-o", "--output", type=str, help="Path for output YAML file") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output YAML file if it already exists") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -def create_test_yml(ctx, tool, run_tests, output, force, no_prompts): +@click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") +def create_snapshot(ctx, tool, run_tests, no_prompts, update): """ - Auto-generate a test.yml file for a new module. + Generate nf-test snapshots for a module. - Given the name of a module, runs the Nextflow test command and automatically generate - the required `test.yml` file based on the output files. + Given the name of a module, runs the nf-test command to generate snapshots. """ - from nf_core.modules import ModulesTestYmlBuilder + from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator try: - meta_builder = ModulesTestYmlBuilder( - module_name=tool, + snap_generator = ComponentTestSnapshotGenerator( + component_name=tool, run_tests=run_tests, - test_yml_output_path=output, - force_overwrite=force, no_prompts=no_prompts, + update=update, remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], + verbose=ctx.obj["verbose"], ) - meta_builder.run() + snap_generator.run() except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) diff --git a/nf_core/list.py b/nf_core/list.py index c6c4bdf756..94d9d8e043 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -128,12 +128,14 @@ def get_local_nf_workflows(self): # Fetch details about local cached pipelines with `nextflow list` else: log.debug("Getting list of local nextflow workflows") - nflist_raw = nf_core.utils.run_cmd("nextflow", "list") - for wf_name in nflist_raw.splitlines(): - if not str(wf_name).startswith("nf-core/"): - self.local_unmatched.append(wf_name) - else: - self.local_workflows.append(LocalWorkflow(wf_name)) + result = nf_core.utils.run_cmd("nextflow", "list") + if result is not None: + nflist_raw, _ = result + for wf_name in nflist_raw.splitlines(): + if not str(wf_name).startswith("nf-core/"): + self.local_unmatched.append(wf_name) + else: + self.local_workflows.append(LocalWorkflow(wf_name)) # Find additional information about each workflow by checking its git history log.debug(f"Fetching extra info about {len(self.local_workflows)} local workflows") @@ -342,12 +344,14 @@ def get_local_nf_workflow_details(self): # Use `nextflow info` to get more details about the workflow else: - nfinfo_raw = str(nf_core.utils.run_cmd("nextflow", f"info -d {self.full_name}")) - re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} - for key, pattern in re_patterns.items(): - m = re.search(pattern, nfinfo_raw) - if m: - setattr(self, key, m.group(1)) + result = nf_core.utils.run_cmd("nextflow", f"info -d {self.full_name}") + if result is not None: + nfinfo_raw, _ = result + re_patterns = {"repository": r"repository\s*: (.*)", "local_path": r"local path\s*: (.*)"} + for key, pattern in re_patterns.items(): + m = re.search(pattern, str(nfinfo_raw)) + if m: + setattr(self, key, m.group(1)) # Pull information from the local git repository if self.local_path is not None: diff --git a/nf_core/utils.py b/nf_core/utils.py index 955ef524bc..25bbfabbb7 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -17,7 +17,7 @@ import sys import time from pathlib import Path -from typing import ByteString, Optional, Union +from typing import Union import git import prompt_toolkit @@ -270,14 +270,16 @@ def fetch_wf_config(wf_path, cache_config=True): log.debug("No config cache found") # Call `nextflow config` - nfconfig_raw = run_cmd("nextflow", f"config -flat {wf_path}") - for l in nfconfig_raw.splitlines(): - ul = l.decode("utf-8") - try: - k, v = ul.split(" = ", 1) - config[k] = v.strip("'\"") - except ValueError: - log.debug(f"Couldn't find key=value config pair:\n {ul}") + result = run_cmd("nextflow", f"config -flat {wf_path}") + if result is not None: + nfconfig_raw, _ = result + for l in nfconfig_raw.splitlines(): + ul = l.decode("utf-8") + try: + k, v = ul.split(" = ", 1) + config[k] = v.strip("'\"") + except ValueError: + log.debug(f"Couldn't find key=value config pair:\n {ul}") # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. @@ -304,22 +306,26 @@ def fetch_wf_config(wf_path, cache_config=True): return config -def run_cmd(executable: str, cmd: str) -> Union[Optional[ByteString], str]: +def run_cmd(executable: str, cmd: str) -> Union[tuple[bytes, bytes], None]: """Run a specified command and capture the output. Handle errors nicely.""" full_cmd = f"{executable} {cmd}" log.debug(f"Running command: {full_cmd}") try: proc = subprocess.run(shlex.split(full_cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - return proc.stdout + return (proc.stdout, proc.stderr) except OSError as e: if e.errno == errno.ENOENT: - raise AssertionError( + raise RuntimeError( f"It looks like {executable} is not installed. Please ensure it is available in your PATH." ) except subprocess.CalledProcessError as e: - raise AssertionError( - f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" - ) + log.debug(f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}") + if executable == "nf-test": + return (e.stdout, e.stderr) + else: + raise RuntimeError( + f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}{e.stdout.decode()}" + ) def setup_nfcore_dir(): From 71c441cf9acab3c610cf12ad6da34e3287b0eb6e Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 1 Nov 2023 16:54:47 +0100 Subject: [PATCH 033/158] install all type specific dependencies in CI step --- .github/workflows/lint-code.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 3cc4dd920b..2a18175922 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -93,7 +93,11 @@ jobs: - uses: actions/setup-python@v3 with: python-version: "3.11" - - run: pip install mypy types-PyYAML + - name: Install dependencies + run: | + python -m pip install --upgrade pip -r requirements-dev.txt + pip install -e . + - name: Get Python changed files id: changed-py-files uses: tj-actions/changed-files@v23 From a7cfdd5e3eb1cc362098af191cab4c3b9f96db0b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 2 Nov 2023 09:25:48 +0100 Subject: [PATCH 034/158] =?UTF-8?q?add=20new=20command=20=F0=9F=99=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nf_core/components/snapshot_generator.py | 174 +++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 nf_core/components/snapshot_generator.py diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py new file mode 100644 index 0000000000..deb1b2aa6c --- /dev/null +++ b/nf_core/components/snapshot_generator.py @@ -0,0 +1,174 @@ +""" +The ComponentTestSnapshotGenerator class handles the generation nf-test snapshots. +""" + +from __future__ import print_function + +import logging +import os +import re + +import questionary +import yaml +from rich import print +from rich.panel import Panel +from rich.prompt import Confirm +from rich.syntax import Syntax +from rich.text import Text + +import nf_core.utils +from nf_core.components.components_command import ComponentCommand + +log = logging.getLogger(__name__) + + +class ComponentTestSnapshotGenerator(ComponentCommand): + def __init__( + self, + component_name=None, + directory=".", + run_tests=False, + force_overwrite=False, + no_prompts=False, + remote_url=None, + branch=None, + verbose=False, + update=False, + ): + super().__init__("modules", directory, remote_url, branch) + self.component_name = component_name + self.remote_url = remote_url + self.branch = branch + self.run_tests = run_tests + self.force_overwrite = force_overwrite + self.no_prompts = no_prompts + self.component_dir = None + self.entry_points = [] + self.tests = [] + self.errors = [] + self.verbose = verbose + self.obsolete_snapshots = False + self.update = update + + def run(self): + """Run build steps""" + self.check_inputs() + self.check_snapshot_stability() + if len(self.errors) > 0: + errors = "\n - ".join(self.errors) + raise UserWarning(f"Ran, but found errors:\n - {errors}") + + def check_inputs(self): + """Do more complex checks about supplied flags.""" + # Check modules directory structure + self.check_modules_structure() + + # Get the tool name if not specified + if self.component_name is None: + self.component_name = questionary.autocomplete( + "Tool name:", + choices=self.components_from_repo(self.org), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + self.component_dir = os.path.join(self.default_modules_path, *self.component_name.split("/")) + + # First, sanity check that the module directory exists + if not os.path.isdir(self.component_dir): + raise UserWarning(f"Cannot find directory '{self.component_dir}'. Should be TOOL/SUBTOOL or TOOL") + + # Check that we're running tests if no prompts + if not self.run_tests and self.no_prompts: + log.debug("Setting run_tests to True as running without prompts") + self.run_tests = True + + # Check container software to use + if os.environ.get("PROFILE") is None: + os.environ["PROFILE"] = "" + if self.no_prompts: + log.info( + "Setting env var '$PROFILE' to an empty string as not set.\n" + "Tests will run with Docker by default. " + "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." + ) + else: + question = { + "type": "list", + "name": "profile", + "message": "Choose container software to run the test with", + "choices": ["Docker", "Singularity", "Conda"], + } + answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) + profile = answer["profile"].lower() + os.environ["PROFILE"] = profile + + def display_nftest_output(self, nftest_out, nftest_err): + nftest_output = Text.from_ansi(nftest_out.decode()) + print(Panel(nftest_output, title="nf-test output")) + if nftest_err: + syntax = Syntax(nftest_err.decode(), "java", theme="ansi_dark") + print(Panel(syntax, title="nf-test error")) + + def generate_snapshot(self) -> bool: + """Generate the nf-test snapshot using `nf-test test` command + + returns True if the test was successful, False otherwise + """ + + log.debug("Running nf-test test") + + # set verbose flag if self.verbose is True + verbose = "--verbose --debug" if self.verbose else "" + update = "--update-snapshot" if self.update else "" + self.update = False # reset self.update to False to test if the new snapshot is stable + + result = nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", + ) + if result is not None: + nftest_out, nftest_err = result + self.display_nftest_output(nftest_out, nftest_err) + # check if nftest_out contains obsolete snapshots + pattern = r"Snapshot Summary:.*?(\d+)\s+obsolete" + compiled_pattern = re.compile(pattern, re.DOTALL) # re.DOTALL to allow . to match newlines + obsolete_snapshots = compiled_pattern.search(nftest_out.decode()) + if obsolete_snapshots: + self.obsolete_snapshots = True + + # check if nf-test was successful + if "Assertion failed:" in nftest_out.decode(): + log.error("nf-test failed") + return False + else: + log.debug("nf-test successful") + return True + else: + log.error("nf-test failed") + return False + + def check_snapshot_stability(self) -> bool: + """Run the nf-test twice and check if the snapshot changes""" + log.info("Generating nf-test snapshot") + if not self.generate_snapshot(): + return False # stop here if the first run failed + log.info("Generating nf-test snapshot again to check stability") + if not self.generate_snapshot(): + log.error("nf-test snapshot is not stable") + return False + else: + if self.obsolete_snapshots: + # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot + if self.no_prompts: + log.info("Removing obsolete snapshots") + nf_core.utils.run_cmd("nf-test", "clean-snapshot") + else: + answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) + if answer: + log.info("Removing obsolete snapshots") + nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + ) + else: + log.debug("Obsolete snapshots not removed") + return True From d6b9f6c0248aaedae288d1785f9b56779de31db5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 2 Nov 2023 09:30:15 +0100 Subject: [PATCH 035/158] remove tests for modules create_test_yml --- nf_core/modules/test_yml_builder.py | 383 ---------------------------- tests/modules/create_test_yml.py | 60 ----- tests/test_modules.py | 7 - 3 files changed, 450 deletions(-) delete mode 100644 nf_core/modules/test_yml_builder.py delete mode 100644 tests/modules/create_test_yml.py diff --git a/nf_core/modules/test_yml_builder.py b/nf_core/modules/test_yml_builder.py deleted file mode 100644 index 092f260fcf..0000000000 --- a/nf_core/modules/test_yml_builder.py +++ /dev/null @@ -1,383 +0,0 @@ -""" -The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file -along with running the tests and creating md5 sums -""" - -from __future__ import print_function - -import errno -import gzip -import hashlib -import io -import logging -import operator -import os -import re -import shlex -import subprocess -import tempfile - -import questionary -import rich -import yaml -from rich.syntax import Syntax - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand - -from ..lint_utils import run_prettier_on_file -from .modules_repo import ModulesRepo - -log = logging.getLogger(__name__) - - -class ModulesTestYmlBuilder(ComponentCommand): - def __init__( - self, - module_name=None, - directory=".", - run_tests=False, - test_yml_output_path=None, - force_overwrite=False, - no_prompts=False, - remote_url=None, - branch=None, - ): - super().__init__("modules", directory, remote_url, branch) - self.module_name = module_name - self.remote_url = remote_url - self.branch = branch - self.run_tests = run_tests - self.test_yml_output_path = test_yml_output_path - self.force_overwrite = force_overwrite - self.no_prompts = no_prompts - self.module_dir = None - self.module_test_main = None - self.entry_points = [] - self.tests = [] - self.errors = [] - - def run(self): - """Run build steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values " - "[cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self.check_inputs() - self.scrape_workflow_entry_points() - self.build_all_tests() - self.print_test_yml() - if len(self.errors) > 0: - errors = "\n - ".join(self.errors) - raise UserWarning(f"Ran, but found errors:\n - {errors}") - - def check_inputs(self): - """Do more complex checks about supplied flags.""" - # Check modules directory structure - self.check_modules_structure() - - # Get the tool name if not specified - if self.module_name is None: - self.module_name = questionary.autocomplete( - "Tool name:", - choices=self.components_from_repo(self.org), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - self.module_dir = os.path.join(self.default_modules_path, *self.module_name.split("/")) - self.module_test_main = os.path.join(self.default_tests_path, *self.module_name.split("/"), "main.nf") - - # First, sanity check that the module directory exists - if not os.path.isdir(self.module_dir): - raise UserWarning(f"Cannot find directory '{self.module_dir}'. Should be TOOL/SUBTOOL or TOOL") - if not os.path.exists(self.module_test_main): - raise UserWarning(f"Cannot find module test workflow '{self.module_test_main}'") - - # Check that we're running tests if no prompts - if not self.run_tests and self.no_prompts: - log.debug("Setting run_tests to True as running without prompts") - self.run_tests = True - - # Get the output YAML file / check it does not already exist - while self.test_yml_output_path is None: - default_val = f"tests/modules/{self.org}/{self.module_name}/test.yml" - if self.no_prompts: - self.test_yml_output_path = default_val - else: - self.test_yml_output_path = rich.prompt.Prompt.ask( - "[violet]Test YAML output path[/] (- for stdout)", default=default_val - ).strip() - if self.test_yml_output_path == "": - self.test_yml_output_path = None - # Check that the output YAML file does not already exist - if ( - self.test_yml_output_path is not None - and self.test_yml_output_path != "-" - and os.path.exists(self.test_yml_output_path) - and not self.force_overwrite - ): - if rich.prompt.Confirm.ask( - f"[red]File exists! [green]'{self.test_yml_output_path}' [violet]Overwrite?" - ): - self.force_overwrite = True - else: - self.test_yml_output_path = None - if os.path.exists(self.test_yml_output_path) and not self.force_overwrite: - raise UserWarning( - f"Test YAML file already exists! '{self.test_yml_output_path}'. Use '--force' to overwrite." - ) - - def scrape_workflow_entry_points(self): - """Find the test workflow entry points from main.nf""" - log.info(f"Looking for test workflow entry points: '{self.module_test_main}'") - with open(self.module_test_main, "r") as fh: - for line in fh: - match = re.match(r"workflow\s+(\S+)\s+{", line) - if match: - self.entry_points.append(match.group(1)) - if len(self.entry_points) == 0: - raise UserWarning("No workflow entry points found in 'self.module_test_main'") - - def build_all_tests(self): - """ - Go over each entry point and build structure - """ - - # Build the other tests - for entry_point in self.entry_points: - ep_test = self.build_single_test(entry_point) - if ep_test: - self.tests.append(ep_test) - - # Build the stub test - stub_test = self.build_single_test(self.entry_points[0], stub=True) - if stub_test: - self.tests.append(stub_test) - - def build_single_test(self, entry_point, stub=False): - """Given the supplied cli flags, prompt for any that are missing. - - Returns: Test command - """ - ep_test = { - "name": "", - "command": "", - "tags": [], - "files": [], - } - stub_option = " -stub" if stub else "" - stub_name = " stub" if stub else "" - - # Print nice divider line - console = rich.console.Console() - console.print("[black]" + "─" * console.width) - - log.info(f"Building test meta for entry point '{entry_point}'") - - while ep_test["name"] == "": - default_val = f"{self.module_name.replace('/', ' ')} {entry_point}{stub_name}" - if self.no_prompts: - ep_test["name"] = default_val - else: - ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() - - while ep_test["command"] == "": - # Don't think we need the last `-c` flag, but keeping to avoid having to update 100s modules. - # See https://github.com/nf-core/tools/issues/1562 - default_val = ( - f"nextflow run ./tests/modules/{self.org}/{self.module_name} -entry {entry_point} " - f"-c ./tests/config/nextflow.config" - f"{stub_option}" - ) - if self.no_prompts: - ep_test["command"] = default_val - else: - ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() - - while len(ep_test["tags"]) == 0: - mod_name_parts = self.module_name.split("/") - tag_defaults = [] - for idx in range(0, len(mod_name_parts)): - tag_defaults.append("/".join(mod_name_parts[: idx + 1])) - # Remove duplicates - tag_defaults = list(set(tag_defaults)) - if self.no_prompts: - ep_test["tags"] = tag_defaults - else: - while len(ep_test["tags"]) == 0: - prompt_tags = rich.prompt.Prompt.ask( - "[violet]Test tags[/] (comma separated)", default=",".join(tag_defaults) - ).strip() - ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] - - ep_test["files"] = self.get_md5_sums(ep_test["command"], stub=stub) - - return ep_test - - def check_if_empty_file(self, fname): - """Check if the file is empty, or compressed empty""" - if os.path.getsize(fname) == 0: - return True - try: - with open(fname, "rb") as fh: - g_f = gzip.GzipFile(fileobj=fh, mode="rb") - if g_f.read() == b"": - return True - except gzip.BadGzipFile: - pass - - return False - - def _md5(self, fname): - """Generate md5 sum for file""" - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b""): - hash_md5.update(chunk) - md5sum = hash_md5.hexdigest() - return md5sum - - def create_test_file_dict(self, results_dir, is_repeat=False, stub=False): - """Walk through directory and collect md5 sums""" - test_files = [] - for root, _, files in os.walk(results_dir, followlinks=True): - for filename in files: - file_path = os.path.join(root, filename) - # add the key here so that it comes first in the dict - test_file = {"path": file_path} - # Check that this isn't an empty file - if self.check_if_empty_file(file_path) and not stub: - if not is_repeat: - self.errors.append(f"Empty file found! '{os.path.basename(file_path)}'") - # Add the md5 anyway, linting should fail later and can be manually removed if needed. - # Originally we skipped this if empty, but then it's too easy to miss the warning. - # Equally, if a file is legitimately empty we don't want to prevent this from working. - if filename != "versions.yml" and not stub: - # Only add md5sum if the file is not versions.yml - file_md5 = self._md5(file_path) - test_file["md5sum"] = file_md5 - # Switch out the results directory path with the expected 'output' directory - test_file["path"] = file_path.replace(results_dir, "output") - test_files.append(test_file) - - test_files = sorted(test_files, key=operator.itemgetter("path")) - - return test_files - - def get_md5_sums(self, command, results_dir=None, results_dir_repeat=None, stub=False): - """ - Recursively go through directories and subdirectories - and generate tuples of (, ) - returns: list of tuples - """ - - run_this_test = False - while results_dir is None: - if self.run_tests or run_this_test: - results_dir, results_dir_repeat = self.run_tests_workflow(command) - else: - results_dir = rich.prompt.Prompt.ask( - "[violet]Test output folder with results[/] (leave blank to run test)" - ) - if results_dir == "": - results_dir = None - run_this_test = True - elif not os.path.isdir(results_dir): - log.error(f"Directory '{results_dir}' does not exist") - results_dir = None - - test_files = self.create_test_file_dict(results_dir=results_dir, stub=stub) - - # If test was repeated, compare the md5 sums - if results_dir_repeat: - test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat, is_repeat=True, stub=stub) - - # Compare both test.yml files - for i in range(len(test_files)): - if test_files[i].get("md5sum") and not test_files[i].get("md5sum") == test_files_repeat[i]["md5sum"]: - test_files[i].pop("md5sum") - test_files[i]["contains"] = [ - "# TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead " - ] - - if len(test_files) == 0: - raise UserWarning(f"Could not find any test result files in '{results_dir}'") - - return test_files - - def run_tests_workflow(self, command): - """Given a test workflow and an entry point, run the test workflow""" - - # The config expects $PROFILE and Nextflow fails if it's not set - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting env var '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - if profile in ["singularity", "conda"]: - os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") - - tmp_dir = tempfile.mkdtemp() - tmp_dir_repeat = tempfile.mkdtemp() - work_dir = tempfile.mkdtemp() - command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" - command += f" --outdir {tmp_dir} -work-dir {work_dir}" - - log.info(f"Running '{self.module_name}' test with command:\n[violet]{command}") - try: - nfconfig_raw = subprocess.check_output(shlex.split(command)) - log.info("Repeating test ...") - nfconfig_raw = subprocess.check_output(shlex.split(command_repeat)) - - except OSError as e: - if e.errno == errno.ENOENT and command.strip().startswith("nextflow "): - raise AssertionError( - "It looks like Nextflow is not installed. It is required for most nf-core functions." - ) - except subprocess.CalledProcessError as e: - output = rich.markup.escape(e.output.decode()) - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") - except Exception as e: - raise UserWarning(f"Error running test workflow: {e}") - else: - log.info("Test workflow finished!") - try: - log.debug(rich.markup.escape(nfconfig_raw)) - except TypeError: - log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) - - return tmp_dir, tmp_dir_repeat - - def print_test_yml(self): - """ - Generate the test yml file. - """ - with tempfile.NamedTemporaryFile(mode="w+") as fh: - yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) - run_prettier_on_file(fh.name) - fh.seek(0) - prettified_yml = fh.read() - - if self.test_yml_output_path == "-": - console = rich.console.Console() - console.print("\n", Syntax(prettified_yml, "yaml"), "\n") - else: - try: - log.info(f"Writing to '{self.test_yml_output_path}'") - with open(self.test_yml_output_path, "w") as fh: - fh.write(prettified_yml) - except FileNotFoundError as e: - raise UserWarning(f"Could not create test.yml file: '{e}'") diff --git a/tests/modules/create_test_yml.py b/tests/modules/create_test_yml.py deleted file mode 100644 index 243378af78..0000000000 --- a/tests/modules/create_test_yml.py +++ /dev/null @@ -1,60 +0,0 @@ -import os -from pathlib import Path - -import pytest - -import nf_core.modules - -from ..utils import GITLAB_DEFAULT_BRANCH, GITLAB_URL, with_temporary_folder - - -@with_temporary_folder -def test_modules_custom_yml_dumper(self, out_dir): - """Try to create a yml file with the custom yml dumper""" - yml_output_path = Path(out_dir, "test.yml") - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) - meta_builder.test_yml_output_path = yml_output_path - meta_builder.tests = [{"testname": "myname"}] - meta_builder.print_test_yml() - assert Path(yml_output_path).is_file() - - -@with_temporary_folder -def test_modules_test_file_dict(self, test_file_dir): - """Create dict of test files and create md5 sums""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) - with open(Path(test_file_dir, "test_file.txt"), "w") as fh: - fh.write("this line is just for testing") - test_files = meta_builder.create_test_file_dict(test_file_dir) - assert len(test_files) == 1 - assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" - - -@with_temporary_folder -def test_modules_create_test_yml_get_md5(self, test_file_dir): - """Get md5 sums from a dummy output""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("test/tool", self.pipeline_dir, False, "./", False, True) - with open(Path(test_file_dir, "test_file.txt"), "w") as fh: - fh.write("this line is just for testing") - test_files = meta_builder.get_md5_sums(command="dummy", results_dir=test_file_dir, results_dir_repeat=test_file_dir) - assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" - - -def test_modules_create_test_yml_entry_points(self): - """Test extracting test entry points from a main.nf file""" - meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", self.pipeline_dir, False, "./", False, True) - meta_builder.module_test_main = Path(self.nfcore_modules, "tests", "modules", "nf-core", "bpipe", "test", "main.nf") - meta_builder.scrape_workflow_entry_points() - assert meta_builder.entry_points[0] == "test_bpipe_test" - - -def test_modules_create_test_yml_check_inputs(self): - """Test the check_inputs() function - raise UserWarning because test.yml exists""" - cwd = os.getcwd() - os.chdir(self.nfcore_modules) - meta_builder = nf_core.modules.ModulesTestYmlBuilder("bpipe/test", ".", False, "./", False, True) - meta_builder.module_test_main = Path(self.nfcore_modules, "tests", "modules", "bpipe", "test", "main.nf") - with pytest.raises(UserWarning) as excinfo: - meta_builder.check_inputs() - os.chdir(cwd) - assert "Test YAML file already exists!" in str(excinfo.value) diff --git a/tests/test_modules.py b/tests/test_modules.py index d4c8b9fc52..58a8b5fade 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -147,13 +147,6 @@ def test_modulesrepo_class(self): test_modules_create_nfcore_modules_subtool, test_modules_create_succeed, ) - from .modules.create_test_yml import ( - test_modules_create_test_yml_check_inputs, - test_modules_create_test_yml_entry_points, - test_modules_create_test_yml_get_md5, - test_modules_custom_yml_dumper, - test_modules_test_file_dict, - ) from .modules.info import ( test_modules_info_in_modules_repo, test_modules_info_local, From 6b0ff4992a9ce9baec1dac1ff548282ea222d42c Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 2 Nov 2023 09:36:27 +0100 Subject: [PATCH 036/158] remove more mentions of test_yml_builder --- nf_core/modules/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 47af637d02..461d813e83 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -10,5 +10,4 @@ from .modules_utils import ModuleException from .patch import ModulePatch from .remove import ModuleRemove -from .test_yml_builder import ModulesTestYmlBuilder from .update import ModuleUpdate From 0045d4d6aeb207308c5e70168cf652e5910c7314 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Nov 2023 12:41:51 +0100 Subject: [PATCH 037/158] add types to create-snapshot and use component_type --- nf_core/__main__.py | 1 + nf_core/components/snapshot_generator.py | 54 +++++++++++++++--------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index dc1f5c4e07..644a3f9c73 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -866,6 +866,7 @@ def create_snapshot(ctx, tool, run_tests, no_prompts, update): try: snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", component_name=tool, run_tests=run_tests, no_prompts=no_prompts, diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index deb1b2aa6c..13b18baeb6 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -7,6 +7,7 @@ import logging import os import re +from typing import List, Optional import questionary import yaml @@ -23,31 +24,34 @@ class ComponentTestSnapshotGenerator(ComponentCommand): + """ + Class to generate nf-test snapshots for modules. + """ + def __init__( self, - component_name=None, - directory=".", - run_tests=False, - force_overwrite=False, - no_prompts=False, - remote_url=None, - branch=None, - verbose=False, - update=False, + component_type: str, + component_name: Optional[str] = None, + directory: str = ".", + run_tests: bool = False, + force_overwrite: bool = False, + no_prompts: bool = False, + remote_url: Optional[str] = None, + branch: Optional[str] = None, + verbose: bool = False, + update: bool = False, ): - super().__init__("modules", directory, remote_url, branch) + super().__init__(component_type, directory, remote_url, branch) self.component_name = component_name self.remote_url = remote_url self.branch = branch self.run_tests = run_tests self.force_overwrite = force_overwrite self.no_prompts = no_prompts - self.component_dir = None - self.entry_points = [] - self.tests = [] - self.errors = [] + self.component_dir: str = None + self.errors: List[str] = [] self.verbose = verbose - self.obsolete_snapshots = False + self.obsolete_snapshots: bool = False self.update = update def run(self): @@ -61,20 +65,25 @@ def run(self): def check_inputs(self): """Do more complex checks about supplied flags.""" # Check modules directory structure - self.check_modules_structure() + if self.component_type == "modules": + self.check_modules_structure() - # Get the tool name if not specified + # Get the component name if not specified if self.component_name is None: self.component_name = questionary.autocomplete( - "Tool name:", + "Tool name:" if self.component_type == "modules" else "Subworkflow name:", choices=self.components_from_repo(self.org), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - self.component_dir = os.path.join(self.default_modules_path, *self.component_name.split("/")) + self.component_dir = os.path.join( + self.component_type, self.modules_repo.repo_path, *self.component_name.split("/") + ) # First, sanity check that the module directory exists if not os.path.isdir(self.component_dir): - raise UserWarning(f"Cannot find directory '{self.component_dir}'. Should be TOOL/SUBTOOL or TOOL") + raise UserWarning( + f"Cannot find directory '{self.component_dir}'.{' Should be TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}" + ) # Check that we're running tests if no prompts if not self.run_tests and self.no_prompts: @@ -160,7 +169,10 @@ def check_snapshot_stability(self) -> bool: # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot if self.no_prompts: log.info("Removing obsolete snapshots") - nf_core.utils.run_cmd("nf-test", "clean-snapshot") + nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + ) else: answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) if answer: From 61459f575418561e0b8919f18a192b3dbd16b0f0 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 2 Nov 2023 13:07:05 +0100 Subject: [PATCH 038/158] add subworkflows create-snapshot --- nf_core/__main__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 644a3f9c73..2ac1c476c0 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1076,34 +1076,34 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): sys.exit(1) -# nf-core subworkflows create-test-yml -@subworkflows.command("create-test-yml") +# nf-core subworkflows create-snapshot +@subworkflows.command("create-snapshot") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") -@click.option("-o", "--output", type=str, help="Path for output YAML file") -@click.option("-f", "--force", is_flag=True, default=False, help="Overwrite output YAML file if it already exists") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -def create_test_yml(ctx, subworkflow, run_tests, output, force, no_prompts): +@click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") +def create_test_yml(ctx, subworkflow, run_tests, no_prompts, update): """ Auto-generate a test.yml file for a new subworkflow. Given the name of a module, runs the Nextflow test command and automatically generate the required `test.yml` file based on the output files. """ - from nf_core.subworkflows import SubworkflowTestYmlBuilder + from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator try: - meta_builder = SubworkflowTestYmlBuilder( - subworkflow=subworkflow, + snap_generator = ComponentTestSnapshotGenerator( + component_type="subworkflows", + component_name=subworkflow, run_tests=run_tests, - test_yml_output_path=output, - force_overwrite=force, no_prompts=no_prompts, + update=update, remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], + verbose=ctx.obj["verbose"], ) - meta_builder.run() + snap_generator.run() except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) From 6b63566360bb843314a9273966695b03667ae095 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 6 Nov 2023 18:12:55 +0100 Subject: [PATCH 039/158] add linting for snapshot file --- .github/workflows/lint-code.yml | 4 +- nf_core/modules/lint/module_tests.py | 12 ++++++ .../subworkflows/lint/subworkflow_tests.py | 10 +++++ tests/modules/lint.py | 33 +++++++++++++++ tests/subworkflows/lint.py | 40 +++++++++++++++++++ tests/test_modules.py | 25 +++++++----- tests/test_subworkflows.py | 27 +++++++++---- 7 files changed, 131 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 2a18175922..c0de5c719d 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -92,7 +92,9 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v3 with: - python-version: "3.11" + python-version: 3.11 + cache: "pip" + - name: Install dependencies run: | python -m pip install --upgrade pip -r requirements-dev.txt diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index ece42c9443..bee01f6b5b 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -40,6 +40,18 @@ def module_tests(_, module): module.failed.append(("test_main_exists", "test `main.nf` does not exist", module.test_main_nf)) module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", nftest_main_nf)) + if os.path.exists(nftest_main_nf): + # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test + with open(nftest_main_nf, "r") as fh: + if "snapshot(" in fh.read(): + snap_file = os.path.join(module.component_dir, "tests", "main.nf.test.snap") + if os.path.exists(snap_file): + module.passed.append(("test_main_snap", "snapshot file `main.nf.test.snap` exists", snap_file)) + else: + module.failed.append( + ("test_main_snap", "snapshot file `main.nf.test.snap` does not exist", snap_file) + ) + if os.path.exists(pytest_main_nf): # Check that entry in pytest_modules.yml exists try: diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index cbbf6e35c4..4515dd21ca 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -45,6 +45,16 @@ def subworkflow_tests(_, subworkflow): subworkflow.failed.append(("test_main_exists", "test `main.nf` does not exist", subworkflow.test_main_nf)) subworkflow.warned.append(("test_main_exists", "test `main.nf.test` does not exist", nftest_main_nf)) + if os.path.exists(nftest_main_nf): + # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test + with open(nftest_main_nf, "r") as fh: + if "snapshot(" in fh.read(): + snap_file = os.path.join(subworkflow.component_dir, "tests", "main.nf.test.snap") + if os.path.exists(snap_file): + subworkflow.passed.append(("test_main_snap", "test `main.nf.test.snap` exists", snap_file)) + else: + subworkflow.failed.append(("test_main_snap", "test `main.nf.test.snap` does not exist", snap_file)) + if os.path.exists(pytest_main_nf): # Check that entry in pytest_modules.yml exists try: diff --git a/tests/modules/lint.py b/tests/modules/lint.py index e26d3e3103..c5d85c4b65 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -306,3 +306,36 @@ def test_modules_lint_check_url(self): assert ( len(mocked_ModuleLint.failed) == failed ), f"{test}: Expected {failed} FAIL, got {len(mocked_ModuleLint.failed)}." + + +def test_modules_lint_snapshot_file(self): + """Test linting a module with a snapshot file""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + +def test_modules_lint_snapshot_file_missing_fail(self): + """Test linting a module with a snapshot file missing, which should fail""" + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + +def test_modules_lint_snapshot_file_not_needed(self): + """Test linting a module which doesn't need a snapshot file by removing the snapshot keyword in the main.nf.test file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "r") as fh: + content = fh.read() + new_content = content.replace("snapshot(", "snap (") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "w") as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index d754985265..36beda5029 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,3 +1,6 @@ +import os +from pathlib import Path + import pytest import nf_core.subworkflows @@ -55,3 +58,40 @@ def test_subworkflows_lint_multiple_remotes(self): assert len(subworkflow_lint.failed) == 1 assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 + + +def test_subworkflows_lint_snapshot_file(self): + """Test linting a subworkflow with a snapshot file""" + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + +def test_subworkflows_lint_snapshot_file_missing_fail(self): + """Test linting a subworkflow with a snapshot file missing, which should fail""" + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + + +def test_subworkflows_lint_snapshot_file_not_needed(self): + """Test linting a subworkflow which doesn't need a snapshot file by removing the snapshot keyword in the main.nf.test file""" + with open( + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test"), "r" + ) as fh: + content = fh.read() + new_content = content.replace("snapshot(", "snap (") + with open( + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test"), "w" + ) as fh: + fh.write(new_content) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 diff --git a/tests/test_modules.py b/tests/test_modules.py index 58a8b5fade..4137163ed5 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -135,25 +135,25 @@ def test_modulesrepo_class(self): # Test of the individual modules commands. # ############################################ - from .modules.bump_versions import ( + from .modules.bump_versions import ( # type: ignore[misc] test_modules_bump_versions_all_modules, test_modules_bump_versions_fail, test_modules_bump_versions_fail_unknown_version, test_modules_bump_versions_single_module, ) - from .modules.create import ( + from .modules.create import ( # type: ignore[misc] test_modules_create_fail_exists, test_modules_create_nfcore_modules, test_modules_create_nfcore_modules_subtool, test_modules_create_succeed, ) - from .modules.info import ( + from .modules.info import ( # type: ignore[misc] test_modules_info_in_modules_repo, test_modules_info_local, test_modules_info_remote, test_modules_info_remote_gitlab, ) - from .modules.install import ( + from .modules.install import ( # type: ignore[misc] test_modules_install_alternate_remote, test_modules_install_different_branch_fail, test_modules_install_different_branch_succeed, @@ -165,7 +165,7 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore, test_modules_install_trimgalore_twice, ) - from .modules.lint import ( + from .modules.lint import ( # type: ignore[misc] test_modules_lint_check_process_labels, test_modules_lint_check_url, test_modules_lint_empty, @@ -174,16 +174,19 @@ def test_modulesrepo_class(self): test_modules_lint_new_modules, test_modules_lint_no_gitlab, test_modules_lint_patched_modules, + test_modules_lint_snapshot_file, + test_modules_lint_snapshot_file_missing_fail, + test_modules_lint_snapshot_file_not_needed, test_modules_lint_trimgalore, ) - from .modules.list import ( + from .modules.list import ( # type: ignore[misc] test_modules_install_and_list_pipeline, test_modules_install_gitlab_and_list_pipeline, test_modules_list_pipeline, test_modules_list_remote, test_modules_list_remote_gitlab, ) - from .modules.modules_json import ( + from .modules.modules_json import ( # type: ignore[misc] test_get_modules_json, test_mod_json_create, test_mod_json_create_with_patch, @@ -198,12 +201,12 @@ def test_modulesrepo_class(self): test_mod_json_with_empty_modules_value, test_mod_json_with_missing_modules_entry, ) - from .modules.modules_test import ( + from .modules.modules_test import ( # type: ignore[misc] test_modules_test_check_inputs, test_modules_test_no_installed_modules, test_modules_test_no_name_no_prompts, ) - from .modules.patch import ( + from .modules.patch import ( # type: ignore[misc] test_create_patch_change, test_create_patch_no_change, test_create_patch_try_apply_failed, @@ -212,12 +215,12 @@ def test_modulesrepo_class(self): test_create_patch_update_success, test_remove_patch, ) - from .modules.remove import ( + from .modules.remove import ( # type: ignore[misc] test_modules_remove_multiqc_from_gitlab, test_modules_remove_trimgalore, test_modules_remove_trimgalore_uninstalled, ) - from .modules.update import ( + from .modules.update import ( # type: ignore[misc] test_install_and_update, test_install_at_hash_and_update, test_install_at_hash_and_update_and_save_diff_to_file, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 1c290cb882..f64c900f10 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -101,25 +101,25 @@ def tearDown(self): # Test of the individual subworkflow commands. # ################################################ - from .subworkflows.create import ( + from .subworkflows.create import ( # type: ignore[misc] test_subworkflows_create_fail_exists, test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) - from .subworkflows.create_test_yml import ( + from .subworkflows.create_test_yml import ( # type: ignore[misc] test_subworkflows_create_test_yml_check_inputs, test_subworkflows_create_test_yml_entry_points, test_subworkflows_create_test_yml_get_md5, test_subworkflows_custom_yml_dumper, test_subworkflows_test_file_dict, ) - from .subworkflows.info import ( + from .subworkflows.info import ( # type: ignore[misc] test_subworkflows_info_in_modules_repo, test_subworkflows_info_local, test_subworkflows_info_remote, test_subworkflows_info_remote_gitlab, ) - from .subworkflows.install import ( + from .subworkflows.install import ( # type: ignore[misc] test_subworkflow_install_nopipeline, test_subworkflows_install_alternate_remote, test_subworkflows_install_bam_sort_stats_samtools, @@ -132,24 +132,35 @@ def tearDown(self): test_subworkflows_install_tracking_added_already_installed, test_subworkflows_install_tracking_added_super_subworkflow, ) - from .subworkflows.list import ( + from .subworkflows.lint import ( # type: ignore[misc] + test_subworkflows_lint, + test_subworkflows_lint_empty, + test_subworkflows_lint_gitlab_subworkflows, + test_subworkflows_lint_multiple_remotes, + test_subworkflows_lint_new_subworkflow, + test_subworkflows_lint_no_gitlab, + test_subworkflows_lint_snapshot_file, + test_subworkflows_lint_snapshot_file_missing_fail, + test_subworkflows_lint_snapshot_file_not_needed, + ) + from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, test_subworkflows_install_gitlab_and_list_subworkflows, test_subworkflows_list_remote, test_subworkflows_list_remote_gitlab, ) - from .subworkflows.remove import ( + from .subworkflows.remove import ( # type: ignore[misc] test_subworkflows_remove_included_subworkflow, test_subworkflows_remove_one_of_two_subworkflow, test_subworkflows_remove_subworkflow, test_subworkflows_remove_subworkflow_keep_installed_module, ) - from .subworkflows.subworkflows_test import ( + from .subworkflows.subworkflows_test import ( # type: ignore[misc] test_subworkflows_test_check_inputs, test_subworkflows_test_no_installed_subworkflows, test_subworkflows_test_no_name_no_prompts, ) - from .subworkflows.update import ( + from .subworkflows.update import ( # type: ignore[misc] test_install_and_update, test_install_at_hash_and_update, test_install_at_hash_and_update_and_save_diff_to_file, From 73487dde6c4f9d2b4e43b7e2df6ac245ae7cc9fa Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 6 Nov 2023 18:13:52 +0100 Subject: [PATCH 040/158] add correct tags to subworkflow template --- nf_core/subworkflow-template/subworkflows/tests/main.nf.test | 2 ++ nf_core/subworkflow-template/subworkflows/tests/tags.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index b95cfeb383..32f26be546 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -9,6 +9,8 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" tag "{{ component_name }}" + tag "subworkflows/{{ component_name }}" + // TODO nf-core: Add tags for all modules used within this subworkflow diff --git a/nf_core/subworkflow-template/subworkflows/tests/tags.yml b/nf_core/subworkflow-template/subworkflows/tests/tags.yml index 8cde627a13..35cad36785 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/tags.yml +++ b/nf_core/subworkflow-template/subworkflows/tests/tags.yml @@ -1,2 +1,2 @@ -{{ component_name_underscore }}: +subworkflows/{{ component_name_underscore }}: - subworkflows/{{ org }}/{{ component_dir }}/** From cdf5f63e8492e1f64c2b213a9276926ae375ec6e Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 6 Nov 2023 18:14:12 +0100 Subject: [PATCH 041/158] fix smaller things according to mypy --- nf_core/__main__.py | 20 ++++++++++++-------- nf_core/utils.py | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index dc1f5c4e07..4dc1f75601 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -652,7 +652,7 @@ def install(ctx, tool, dir, prompt, force, sha): ctx.obj["modules_repo_no_pull"], ) exit_status = module_install.install(tool) - if not exit_status and all: + if not exit_status: sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) @@ -673,7 +673,9 @@ def install(ctx, tool, dir, prompt, force, sha): @click.option("-f", "--force", is_flag=True, default=False, help="Force update of module") @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") -@click.option("-a", "--all", is_flag=True, default=False, help="Update all modules installed in pipeline") +@click.option( + "-a", "--all", "install_all", is_flag=True, default=False, help="Update all modules installed in pipeline" +) @click.option( "-x/-y", "--preview/--no-preview", @@ -696,7 +698,7 @@ def install(ctx, tool, dir, prompt, force, sha): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_deps): +def update(ctx, tool, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 modules within a pipeline. @@ -719,7 +721,7 @@ def update(ctx, tool, dir, force, prompt, sha, all, preview, save_diff, update_d ctx.obj["modules_repo_no_pull"], ) exit_status = module_install.update(tool) - if not exit_status and all: + if not exit_status and install_all: sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) @@ -1342,7 +1344,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): ctx.obj["modules_repo_no_pull"], ) exit_status = subworkflow_install.install(subworkflow) - if not exit_status and all: + if not exit_status and install_all: sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) @@ -1394,7 +1396,9 @@ def remove(ctx, dir, subworkflow): @click.option("-f", "--force", is_flag=True, default=False, help="Force update of subworkflow") @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the subworkflow") @click.option("-s", "--sha", type=str, metavar="", help="Install subworkflow at commit SHA") -@click.option("-a", "--all", is_flag=True, default=False, help="Update all subworkflow installed in pipeline") +@click.option( + "-a", "--all", "install_all", is_flag=True, default=False, help="Update all subworkflow installed in pipeline" +) @click.option( "-x/-y", "--preview/--no-preview", @@ -1417,7 +1421,7 @@ def remove(ctx, dir, subworkflow): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, update_deps): +def update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 subworkflow within a pipeline. @@ -1440,7 +1444,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, all, preview, save_diff, u ctx.obj["modules_repo_no_pull"], ) exit_status = subworkflow_install.update(subworkflow) - if not exit_status and all: + if not exit_status and install_all: sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) diff --git a/nf_core/utils.py b/nf_core/utils.py index 25bbfabbb7..ee341822a9 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -318,6 +318,8 @@ def run_cmd(executable: str, cmd: str) -> Union[tuple[bytes, bytes], None]: raise RuntimeError( f"It looks like {executable} is not installed. Please ensure it is available in your PATH." ) + else: + return None except subprocess.CalledProcessError as e: log.debug(f"Command '{full_cmd}' returned non-zero error code '{e.returncode}':\n[red]> {e.stderr.decode()}") if executable == "nf-test": From bff7e89b2102500bbca4d7c6b0631a39c9bf7e77 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Nov 2023 11:31:15 +0100 Subject: [PATCH 042/158] add pytests for components snapshot generator --- nf_core/components/snapshot_generator.py | 13 ++++---- tests/components/create_snapshot.py | 41 +++++++++++++++++++++++ tests/test_components.py | 42 ++++++++++++++++++++++++ tests/utils.py | 1 + 4 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 tests/components/create_snapshot.py create mode 100644 tests/test_components.py diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index 13b18baeb6..54f720aa32 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -7,6 +7,7 @@ import logging import os import re +from pathlib import Path from typing import List, Optional import questionary @@ -75,12 +76,10 @@ def check_inputs(self): choices=self.components_from_repo(self.org), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - self.component_dir = os.path.join( - self.component_type, self.modules_repo.repo_path, *self.component_name.split("/") - ) + self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) # First, sanity check that the module directory exists - if not os.path.isdir(self.component_dir): + if not (self.dir / self.component_dir).is_dir(): raise UserWarning( f"Cannot find directory '{self.component_dir}'.{' Should be TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}" ) @@ -132,7 +131,7 @@ def generate_snapshot(self) -> bool: result = nf_core.utils.run_cmd( "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", + f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", ) if result is not None: nftest_out, nftest_err = result @@ -171,7 +170,7 @@ def check_snapshot_stability(self) -> bool: log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", ) else: answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) @@ -179,7 +178,7 @@ def check_snapshot_stability(self) -> bool: log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", ) else: log.debug("Obsolete snapshots not removed") diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py new file mode 100644 index 0000000000..c5829d570b --- /dev/null +++ b/tests/components/create_snapshot.py @@ -0,0 +1,41 @@ +import os +from pathlib import Path + +from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator + +from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL + + +def test_generate_snapshot_module(self): + """Generate the snapshot for a module in nf-core/modules clone""" + for root, directories, files in os.walk(self.nfcore_modules): + print(f"Directory: {root}") + for file in files: + print(f" File: {file}") + + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="fastqc", + no_prompts=True, + directory=self.nfcore_modules, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + assert snap_generator.run() + + assert os.path.exists(Path(self.nfcore_modules, "modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap")) + + +def test_update_snapshot_module(self): + """Update the snapshot of a module in nf-core/modules clone""" + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="bwa/mem", + no_prompts=True, + update=True, + ) + assert snap_generator.run() + + assert os.path.exists( + Path(self.nfcore_modules, "modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") + ) diff --git a/tests/test_components.py b/tests/test_components.py new file mode 100644 index 0000000000..a3b6bf1229 --- /dev/null +++ b/tests/test_components.py @@ -0,0 +1,42 @@ +""" Tests covering the modules commands +""" + +import os +import shutil +import tempfile +import unittest +from pathlib import Path + +from git import Repo + +from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL + + +class TestComponents(unittest.TestCase): + """Class for components tests""" + + def setUp(self): + """Clone a testing version the nf-core/modules repo""" + self.tmp_dir = tempfile.mkdtemp() + + Repo.clone_from(GITLAB_URL, Path(self.tmp_dir, "modules-test"), branch=GITLAB_NFTEST_BRANCH) + + self.nfcore_modules = Path(self.tmp_dir, "modules-test") + + # Set $PROFILE environment variable to docker - tests will run with Docker + os.environ["PROFILE"] = "docker" + + def tearDown(self): + """Clean up temporary files and folders""" + + if self.tmp_dir.is_dir(): + shutil.rmtree(self.tmp_dir) + + ############################################ + # Test of the individual components commands. # + ############################################ + + from .components.create_snapshot import ( # type: ignore[misc] + test_generate_snapshot_module, + test_update_snapshot_module, + ) diff --git a/tests/utils.py b/tests/utils.py index d39d172a66..f258e191ff 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,6 +25,7 @@ GITLAB_BRANCH_ORG_PATH_BRANCH = "org-path" GITLAB_BRANCH_TEST_OLD_SHA = "e772abc22c1ff26afdf377845c323172fb3c19ca" GITLAB_BRANCH_TEST_NEW_SHA = "7d73e21f30041297ea44367f2b4fd4e045c0b991" +GITLAB_NFTEST_BRANCH = "nf-test-tests" def with_temporary_folder(func): From 2f5312af1566a51fd83ccfaf86d7b4f6dd5fe6d0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 7 Nov 2023 12:28:21 +0100 Subject: [PATCH 043/158] add more typing --- nf_core/components/snapshot_generator.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index 13b18baeb6..8d87a9ba56 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -10,7 +10,6 @@ from typing import List, Optional import questionary -import yaml from rich import print from rich.panel import Panel from rich.prompt import Confirm @@ -34,7 +33,6 @@ def __init__( component_name: Optional[str] = None, directory: str = ".", run_tests: bool = False, - force_overwrite: bool = False, no_prompts: bool = False, remote_url: Optional[str] = None, branch: Optional[str] = None, @@ -46,23 +44,24 @@ def __init__( self.remote_url = remote_url self.branch = branch self.run_tests = run_tests - self.force_overwrite = force_overwrite self.no_prompts = no_prompts - self.component_dir: str = None + self.component_dir: str = directory self.errors: List[str] = [] self.verbose = verbose self.obsolete_snapshots: bool = False self.update = update - def run(self): + def run(self) -> None: """Run build steps""" self.check_inputs() self.check_snapshot_stability() if len(self.errors) > 0: errors = "\n - ".join(self.errors) raise UserWarning(f"Ran, but found errors:\n - {errors}") + else: + log.info("All tests passed!") - def check_inputs(self): + def check_inputs(self) -> None: """Do more complex checks about supplied flags.""" # Check modules directory structure if self.component_type == "modules": @@ -75,9 +74,10 @@ def check_inputs(self): choices=self.components_from_repo(self.org), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - self.component_dir = os.path.join( - self.component_type, self.modules_repo.repo_path, *self.component_name.split("/") - ) + if self.component_dir == "": + self.component_dir = os.path.join( + self.component_type, self.modules_repo.repo_path, *self.component_name.split("/") + ) # First, sanity check that the module directory exists if not os.path.isdir(self.component_dir): @@ -95,10 +95,10 @@ def check_inputs(self): os.environ["PROFILE"] = "" if self.no_prompts: log.info( - "Setting env var '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " + "Setting env var '$PROFILE' to Docker as not set.\n" "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." ) + os.environ["PROFILE"] = "docker" else: question = { "type": "list", @@ -110,7 +110,7 @@ def check_inputs(self): profile = answer["profile"].lower() os.environ["PROFILE"] = profile - def display_nftest_output(self, nftest_out, nftest_err): + def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: nftest_output = Text.from_ansi(nftest_out.decode()) print(Panel(nftest_output, title="nf-test output")) if nftest_err: From f42fe20c916e381f862a2d83560fe1e923a58855 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Nov 2023 14:37:18 +0100 Subject: [PATCH 044/158] run create-snapshot tests from modules root directory --- tests/components/create_snapshot.py | 26 ++++++++++++++------------ tests/test_components.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index c5829d570b..ee40e60fa7 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -8,34 +8,36 @@ def test_generate_snapshot_module(self): """Generate the snapshot for a module in nf-core/modules clone""" - for root, directories, files in os.walk(self.nfcore_modules): - print(f"Directory: {root}") - for file in files: - print(f" File: {file}") - + os.chdir(self.nfcore_modules) snap_generator = ComponentTestSnapshotGenerator( component_type="modules", component_name="fastqc", no_prompts=True, - directory=self.nfcore_modules, remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - assert snap_generator.run() + try: + snap_generator.run() + except UserWarning as e: + assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - assert os.path.exists(Path(self.nfcore_modules, "modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap")) + assert Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap").exists() def test_update_snapshot_module(self): """Update the snapshot of a module in nf-core/modules clone""" + os.chdir(self.nfcore_modules) snap_generator = ComponentTestSnapshotGenerator( component_type="modules", component_name="bwa/mem", no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, update=True, ) - assert snap_generator.run() + try: + snap_generator.run() + except UserWarning as e: + assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - assert os.path.exists( - Path(self.nfcore_modules, "modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") - ) + assert Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap").exists() diff --git a/tests/test_components.py b/tests/test_components.py index a3b6bf1229..a419f22de2 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -9,6 +9,8 @@ from git import Repo +from nf_core.modules.modules_repo import ModulesRepo + from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL @@ -17,18 +19,19 @@ class TestComponents(unittest.TestCase): def setUp(self): """Clone a testing version the nf-core/modules repo""" - self.tmp_dir = tempfile.mkdtemp() - - Repo.clone_from(GITLAB_URL, Path(self.tmp_dir, "modules-test"), branch=GITLAB_NFTEST_BRANCH) - + self.tmp_dir = Path(tempfile.mkdtemp()) self.nfcore_modules = Path(self.tmp_dir, "modules-test") + Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_NFTEST_BRANCH) + # Set $PROFILE environment variable to docker - tests will run with Docker - os.environ["PROFILE"] = "docker" + if not os.environ["PROFILE"]: + os.environ["PROFILE"] = "docker" def tearDown(self): """Clean up temporary files and folders""" + # Clean up temporary files if self.tmp_dir.is_dir(): shutil.rmtree(self.tmp_dir) From e79d4d3f07f19aeec9d8cc5beca79a9d28a47449 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 7 Nov 2023 14:45:18 +0100 Subject: [PATCH 045/158] fix checking if PROFILE exists --- tests/test_components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_components.py b/tests/test_components.py index a419f22de2..79276db940 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -25,7 +25,7 @@ def setUp(self): Repo.clone_from(GITLAB_URL, self.nfcore_modules, branch=GITLAB_NFTEST_BRANCH) # Set $PROFILE environment variable to docker - tests will run with Docker - if not os.environ["PROFILE"]: + if os.environ.get("PROFILE") is None: os.environ["PROFILE"] = "docker" def tearDown(self): From c7bc5b70e93bcc0c5d82c91c9da5d4877ba6f892 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 Nov 2023 07:20:27 +0100 Subject: [PATCH 046/158] add new environment.yml setup to template according to #2495 closes #2495 --- nf_core/module-template/modules/environment.yml | 6 ++++++ nf_core/module-template/modules/main.nf | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 nf_core/module-template/modules/environment.yml diff --git a/nf_core/module-template/modules/environment.yml b/nf_core/module-template/modules/environment.yml new file mode 100644 index 0000000000..1594827bae --- /dev/null +++ b/nf_core/module-template/modules/environment.yml @@ -0,0 +1,6 @@ +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 5ffef06bb6..bf0d06059a 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -27,7 +27,7 @@ process {{ component_name_underscore|upper }} { // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. {% endif -%} - conda "{{ bioconda if bioconda else 'YOUR-TOOL-HERE' }}" + conda '${modulesDir}/environment.yml' container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'biocontainers/YOUR-TOOL-HERE' }}' }" From 53b23b7253b78e5d6d5cacf1b8c730e6577e6e70 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Nov 2023 08:59:12 +0100 Subject: [PATCH 047/158] use set_wd for tests --- tests/components/create_snapshot.py | 60 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index ee40e60fa7..ef332705a1 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -3,41 +3,41 @@ from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator -from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL +from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL, set_wd def test_generate_snapshot_module(self): """Generate the snapshot for a module in nf-core/modules clone""" - os.chdir(self.nfcore_modules) - snap_generator = ComponentTestSnapshotGenerator( - component_type="modules", - component_name="fastqc", - no_prompts=True, - remote_url=GITLAB_URL, - branch=GITLAB_NFTEST_BRANCH, - ) - try: - snap_generator.run() - except UserWarning as e: - assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - - assert Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap").exists() + with set_wd(self.nfcore_modules): + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="fastqc", + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + try: + snap_generator.run() + except UserWarning as e: + assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + + assert Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap").exists() def test_update_snapshot_module(self): """Update the snapshot of a module in nf-core/modules clone""" - os.chdir(self.nfcore_modules) - snap_generator = ComponentTestSnapshotGenerator( - component_type="modules", - component_name="bwa/mem", - no_prompts=True, - remote_url=GITLAB_URL, - branch=GITLAB_NFTEST_BRANCH, - update=True, - ) - try: - snap_generator.run() - except UserWarning as e: - assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - - assert Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap").exists() + with set_wd(self.nfcore_modules): + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="bwa/mem", + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + update=True, + ) + try: + snap_generator.run() + except UserWarning as e: + assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + + assert Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap").exists() From c4683f01257dfba6585dd6dfb6648519aa1c2c07 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Nov 2023 11:13:05 +0100 Subject: [PATCH 048/158] check md5sums and add snapshot test for a subworkflow --- tests/components/create_snapshot.py | 52 +++++++++++++++++++++++++++-- tests/test_components.py | 1 + 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index ef332705a1..df5f7c56cb 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -1,3 +1,4 @@ +import json import os from pathlib import Path @@ -21,11 +22,52 @@ def test_generate_snapshot_module(self): except UserWarning as e: assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - assert Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap").exists() + snap_path = Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap") + assert snap_path.exists() + + with open(snap_path, "r") as fh: + snap_content = json.load(fh) + assert "versions" in snap_content + assert "content" in snap_content["versions"] + assert "versions.yml:md5,e1cc25ca8af856014824abd842e93978" in snap_content["versions"]["content"][0] + + +def test_generate_snapshot_subworkflow(self): + """Generate the snapshot for a subworkflows in nf-core/modules clone""" + with set_wd(self.nfcore_modules): + snap_generator = ComponentTestSnapshotGenerator( + component_type="subworkflows", + component_name="bam_sort_stats_samtools", + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + try: + snap_generator.run() + except UserWarning as e: + assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + + snap_path = Path("subworkflows", "nf-core-test", "bam_sort_stats_samtools", "tests", "main.nf.test.snap") + assert snap_path.exists() + + with open(snap_path, "r") as fh: + snap_content = json.load(fh) + assert "test_bam_sort_stats_samtools_paired_end_flagstats" in snap_content + assert ( + "test.flagstat:md5,4f7ffd1e6a5e85524d443209ac97d783" + in snap_content["test_bam_sort_stats_samtools_paired_end_flagstats"]["content"][0][0] + ) + assert "test_bam_sort_stats_samtools_paired_end_idxstats" in snap_content + assert ( + "test.idxstats:md5,df60a8c8d6621100d05178c93fb053a2" + in snap_content["test_bam_sort_stats_samtools_paired_end_idxstats"]["content"][0][0] + ) def test_update_snapshot_module(self): """Update the snapshot of a module in nf-core/modules clone""" + original_timestamp = "2023-10-18T11:02:55.420631681" + with set_wd(self.nfcore_modules): snap_generator = ComponentTestSnapshotGenerator( component_type="modules", @@ -40,4 +82,10 @@ def test_update_snapshot_module(self): except UserWarning as e: assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" - assert Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap").exists() + snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") + assert snap_path.exists() + + with open(snap_path, "r") as fh: + snap_content = json.load(fh) + assert "Single-End" in snap_content + assert snap_content["Single-End"]["timestamp"] != original_timestamp diff --git a/tests/test_components.py b/tests/test_components.py index 79276db940..2a373b8f92 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -41,5 +41,6 @@ def tearDown(self): from .components.create_snapshot import ( # type: ignore[misc] test_generate_snapshot_module, + test_generate_snapshot_subworkflow, test_update_snapshot_module, ) From 2a640cc4afc21b42a04f54aafe998f000ab83ff0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 8 Nov 2023 16:43:52 +0100 Subject: [PATCH 049/158] append error messages to self.errors --- nf_core/components/snapshot_generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index 252cefdd72..9bef9c8576 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -145,7 +145,6 @@ def generate_snapshot(self) -> bool: # check if nf-test was successful if "Assertion failed:" in nftest_out.decode(): - log.error("nf-test failed") return False else: log.debug("nf-test successful") @@ -158,10 +157,13 @@ def check_snapshot_stability(self) -> bool: """Run the nf-test twice and check if the snapshot changes""" log.info("Generating nf-test snapshot") if not self.generate_snapshot(): + log.error("nf-test failed") + self.errors.append("nf-test failed") return False # stop here if the first run failed log.info("Generating nf-test snapshot again to check stability") if not self.generate_snapshot(): log.error("nf-test snapshot is not stable") + self.errors.append("nf-test snapshot is not stable") return False else: if self.obsolete_snapshots: From fa0c6f3f6c17519dce68c866b0ebfe7f4d23bbaf Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 8 Nov 2023 16:59:54 +0100 Subject: [PATCH 050/158] add --dir option to create-snapshot command --- nf_core/__main__.py | 8 ++++++-- nf_core/components/snapshot_generator.py | 15 ++++++++------- tests/utils.py | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 1892cc11cd..5de247e31e 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -855,10 +855,11 @@ def create_module( @modules.command("create-snapshot") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") +@click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") -def create_snapshot(ctx, tool, run_tests, no_prompts, update): +def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update): """ Generate nf-test snapshots for a module. @@ -870,6 +871,7 @@ def create_snapshot(ctx, tool, run_tests, no_prompts, update): snap_generator = ComponentTestSnapshotGenerator( component_type="modules", component_name=tool, + directory=dir, run_tests=run_tests, no_prompts=no_prompts, update=update, @@ -1082,10 +1084,11 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @subworkflows.command("create-snapshot") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") -def create_test_yml(ctx, subworkflow, run_tests, no_prompts, update): +def create_test_yml(ctx, subworkflow, dir, run_tests, no_prompts, update): """ Auto-generate a test.yml file for a new subworkflow. @@ -1098,6 +1101,7 @@ def create_test_yml(ctx, subworkflow, run_tests, no_prompts, update): snap_generator = ComponentTestSnapshotGenerator( component_type="subworkflows", component_name=subworkflow, + directory=dir, run_tests=run_tests, no_prompts=no_prompts, update=update, diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index 9bef9c8576..f2a425c43b 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -19,6 +19,7 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand +from tests.utils import set_wd log = logging.getLogger(__name__) @@ -46,7 +47,6 @@ def __init__( self.branch = branch self.run_tests = run_tests self.no_prompts = no_prompts - self.component_dir: str | Path = directory self.errors: List[str] = [] self.verbose = verbose self.obsolete_snapshots: bool = False @@ -55,7 +55,8 @@ def __init__( def run(self) -> None: """Run build steps""" self.check_inputs() - self.check_snapshot_stability() + with set_wd(self.dir): + self.check_snapshot_stability() if len(self.errors) > 0: errors = "\n - ".join(self.errors) raise UserWarning(f"Ran, but found errors:\n - {errors}") @@ -75,8 +76,8 @@ def check_inputs(self) -> None: choices=self.components_from_repo(self.org), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - if self.component_dir == "": - self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) + + self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) # First, sanity check that the module directory exists if not Path(self.dir, self.component_dir).is_dir(): @@ -131,7 +132,7 @@ def generate_snapshot(self) -> bool: result = nf_core.utils.run_cmd( "nf-test", - f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", ) if result is not None: nftest_out, nftest_err = result @@ -172,7 +173,7 @@ def check_snapshot_stability(self) -> bool: log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", - f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", ) else: answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) @@ -180,7 +181,7 @@ def check_snapshot_stability(self) -> bool: log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", - f"test {self.dir} --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", ) else: log.debug("Obsolete snapshots not removed") diff --git a/tests/utils.py b/tests/utils.py index f258e191ff..5650d6d2eb 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -68,7 +68,7 @@ def set_wd(path: Path): Path to the working directory to be used iside this context. """ start_wd = Path().absolute() - os.chdir(path) + os.chdir(Path(path).resolve()) try: yield finally: From f5ddc64b620deb67456bc0038c4051f118f7c444 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Nov 2023 12:37:22 +0100 Subject: [PATCH 051/158] add no test found error and test for this --- nf_core/components/snapshot_generator.py | 7 ++++-- tests/components/create_snapshot.py | 32 +++++++++++++++--------- tests/test_components.py | 1 + 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index f2a425c43b..c0b1ae2805 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -147,19 +147,22 @@ def generate_snapshot(self) -> bool: # check if nf-test was successful if "Assertion failed:" in nftest_out.decode(): return False + elif "no valid tests found." in nftest_out.decode(): + log.error("Test file 'main.nf.test' not found") + self.errors.append("Test file 'main.nf.test' not found") + return False else: log.debug("nf-test successful") return True else: log.error("nf-test failed") + self.errors.append("nf-test failed") return False def check_snapshot_stability(self) -> bool: """Run the nf-test twice and check if the snapshot changes""" log.info("Generating nf-test snapshot") if not self.generate_snapshot(): - log.error("nf-test failed") - self.errors.append("nf-test failed") return False # stop here if the first run failed log.info("Generating nf-test snapshot again to check stability") if not self.generate_snapshot(): diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index df5f7c56cb..40f585d914 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -2,6 +2,8 @@ import os from pathlib import Path +import pytest + from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL, set_wd @@ -17,10 +19,7 @@ def test_generate_snapshot_module(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - try: - snap_generator.run() - except UserWarning as e: - assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + assert snap_generator.run() snap_path = Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap") assert snap_path.exists() @@ -42,10 +41,7 @@ def test_generate_snapshot_subworkflow(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - try: - snap_generator.run() - except UserWarning as e: - assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + assert snap_generator.run() snap_path = Path("subworkflows", "nf-core-test", "bam_sort_stats_samtools", "tests", "main.nf.test.snap") assert snap_path.exists() @@ -77,10 +73,7 @@ def test_update_snapshot_module(self): branch=GITLAB_NFTEST_BRANCH, update=True, ) - try: - snap_generator.run() - except UserWarning as e: - assert False, f"'ComponentTestSnapshotGenerator' raised an exception {e}" + assert snap_generator.run() snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") assert snap_path.exists() @@ -89,3 +82,18 @@ def test_update_snapshot_module(self): snap_content = json.load(fh) assert "Single-End" in snap_content assert snap_content["Single-End"]["timestamp"] != original_timestamp + + +def test_test_not_found(self): + """Generate the snapshot for a module in nf-core/modules clone""" + with set_wd(self.nfcore_modules): + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="fastp", + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + with pytest.raises(UserWarning) as e: + snap_generator.run() + assert "Test file 'main.nf.test' not found" in e diff --git a/tests/test_components.py b/tests/test_components.py index 2a373b8f92..f630948303 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -42,5 +42,6 @@ def tearDown(self): from .components.create_snapshot import ( # type: ignore[misc] test_generate_snapshot_module, test_generate_snapshot_subworkflow, + test_test_not_found, test_update_snapshot_module, ) From 163019d2d78fe6370e863c1e91736f59e0139678 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Nov 2023 12:41:18 +0100 Subject: [PATCH 052/158] add test for unstable snapshot --- tests/components/create_snapshot.py | 17 ++++++++++++++++- tests/test_components.py | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index 40f585d914..17058f6e17 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -85,7 +85,7 @@ def test_update_snapshot_module(self): def test_test_not_found(self): - """Generate the snapshot for a module in nf-core/modules clone""" + """Generate the snapshot for a module in nf-core/modules clone which diesn't contain tests""" with set_wd(self.nfcore_modules): snap_generator = ComponentTestSnapshotGenerator( component_type="modules", @@ -97,3 +97,18 @@ def test_test_not_found(self): with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in e + + +def test_unstable_snapshot(self): + """Generate the snapshot for a module in nf-core/modules clone with unstable snapshots""" + with set_wd(self.nfcore_modules): + snap_generator = ComponentTestSnapshotGenerator( + component_type="modules", + component_name="kallisto/quant", + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + with pytest.raises(UserWarning) as e: + snap_generator.run() + assert "nf-test snapshot is not stable" in e diff --git a/tests/test_components.py b/tests/test_components.py index f630948303..d2b0c4630f 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -43,5 +43,6 @@ def tearDown(self): test_generate_snapshot_module, test_generate_snapshot_subworkflow, test_test_not_found, + test_unstable_snapshot, test_update_snapshot_module, ) From 8ebf36a52695f406c8b19140c05bd1dc654d6c4f Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 9 Nov 2023 13:28:39 +0100 Subject: [PATCH 053/158] don't assert run() function from snapshot generator --- tests/components/create_snapshot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index 17058f6e17..7d48df696c 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -19,7 +19,7 @@ def test_generate_snapshot_module(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - assert snap_generator.run() + snap_generator.run() snap_path = Path("modules", "nf-core-test", "fastqc", "tests", "main.nf.test.snap") assert snap_path.exists() @@ -41,7 +41,7 @@ def test_generate_snapshot_subworkflow(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) - assert snap_generator.run() + snap_generator.run() snap_path = Path("subworkflows", "nf-core-test", "bam_sort_stats_samtools", "tests", "main.nf.test.snap") assert snap_path.exists() @@ -73,7 +73,7 @@ def test_update_snapshot_module(self): branch=GITLAB_NFTEST_BRANCH, update=True, ) - assert snap_generator.run() + snap_generator.run() snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") assert snap_path.exists() From 749ba53240ac5385574c4b37cb2f45be4efb3e99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 9 Nov 2023 13:48:20 +0100 Subject: [PATCH 054/158] Update tests/components/create_snapshot.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- tests/components/create_snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index 7d48df696c..9e2482baf2 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -85,7 +85,7 @@ def test_update_snapshot_module(self): def test_test_not_found(self): - """Generate the snapshot for a module in nf-core/modules clone which diesn't contain tests""" + """Generate the snapshot for a module in nf-core/modules clone which doesn't contain tests""" with set_wd(self.nfcore_modules): snap_generator = ComponentTestSnapshotGenerator( component_type="modules", From b1b57d37d1e695ad3b9c07ea6437d51ebb2960f8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 12:20:05 +0100 Subject: [PATCH 055/158] add linting code for environemnt.yml --- nf_core/components/create.py | 4 ++ nf_core/components/nfcore_component.py | 1 + .../module-template/modules/environment.yml | 1 + nf_core/modules/lint/__init__.py | 1 + nf_core/modules/lint/environment_yml.py | 67 +++++++++++++++++++ 5 files changed, 74 insertions(+) create mode 100644 nf_core/modules/lint/environment_yml.py diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 64d86be2e8..42e769a340 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -85,6 +85,7 @@ def create(self): modules/nf-core/tool/subtool/ ├── main.nf ├── meta.yml + ├── environment.yml └── tests ├── main.nf.test └── tags.yml @@ -376,6 +377,9 @@ def _get_component_dirs(self): # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") + file_paths[os.path.join(self.component_type, "environment.yml")] = os.path.join( + component_dir, "environment.yml" + ) file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( component_dir, "tests", "tags.yml" ) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 4413854128..0295d7c90d 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -46,6 +46,7 @@ def __init__( # Initialize the important files self.main_nf = self.component_dir / "main.nf" self.meta_yml = self.component_dir / "meta.yml" + self.environment_yml = self.component_dir / "environment.yml" repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1] self.org = repo_dir diff --git a/nf_core/module-template/modules/environment.yml b/nf_core/module-template/modules/environment.yml index 1594827bae..353dcf2e34 100644 --- a/nf_core/module-template/modules/environment.yml +++ b/nf_core/module-template/modules/environment.yml @@ -1,3 +1,4 @@ +name: "{{ component }}" channels: - conda-forge - bioconda diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 83d583dffb..22f222c597 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -29,6 +29,7 @@ class ModuleLint(ComponentLint): """ # Import lint functions + from .environment_yml import environment_yml from .main_nf import main_nf from .meta_yml import meta_yml from .module_changes import module_changes diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py new file mode 100644 index 0000000000..e475ef0e46 --- /dev/null +++ b/nf_core/modules/lint/environment_yml.py @@ -0,0 +1,67 @@ +import json +from pathlib import Path + +import yaml + +from nf_core.components.lint import ComponentLint +from nf_core.components.nfcore_component import NFCoreComponent + + +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent): + """ + Lint an ``environment.yml`` file. + + The lint test checks that the ``dependencies`` section + in the environment.yml file is valid YAML and that it + is sorted alphabetically. + """ + env_yml = None + # load the environment.yml file + try: + with open(Path(module.component_dir, "environment.yml"), "r") as fh: + env_yml = yaml.safe_load(fh) + + module.passed.append(("environment_yml_exists", "Module `environment.yml` exists", module.environment_yml)) + + except FileNotFoundError: + module.failed.append( + ("environment_yml_exists", "Module `environment.yml` does not exist", module.environment_yml) + ) + return + + # Check that the dependencies section is sorted alphabetically + if env_yml: + if "dependencies" in env_yml: + if isinstance(env_yml["dependencies"], list): + if sorted(env_yml["dependencies"]) == env_yml["dependencies"]: + module.passed.append( + ( + "environment_yml_sorted", + "Module's `environment.yml` is sorted alphabetically", + module.environment_yml, + ) + ) + else: + module.failed.append( + ( + "environment_yml_sorted", + "Module's `environment.yml` is not sorted alphabetically", + module.environment_yml, + ) + ) + else: + module.failed.append( + ( + "environment_yml_valid", + "Module's `environment.yml` doesn't have a correctly formatted `dependencies` section, expecting an array", + module.environment_yml, + ) + ) + else: + module.failed.append( + ( + "environment_yml_valid", + "Module's `environment.yml` doesn't contain the required `dependencies` section", + module.environment_yml, + ) + ) From 5aad9c4d806e57a6ecfc274abb90a2b4b2a08970 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 15:12:04 +0100 Subject: [PATCH 056/158] add hint for language server, update to new new for schema file --- nf_core/module-template/modules/environment.yml | 2 ++ nf_core/module-template/modules/meta.yml | 2 +- nf_core/modules/lint/meta_yml.py | 4 ++-- .../modules/nf-core/custom/dumpsoftwareversions/meta.yml | 2 +- nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml | 2 +- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/nf_core/module-template/modules/environment.yml b/nf_core/module-template/modules/environment.yml index 353dcf2e34..61a36cd2e3 100644 --- a/nf_core/module-template/modules/environment.yml +++ b/nf_core/module-template/modules/environment.yml @@ -1,3 +1,5 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json name: "{{ component }}" channels: - conda-forge diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index c5d9c042b4..c27d21e563 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: "{{ component_name_underscore }}" {% if not_empty_template -%} ## TODO nf-core: Add a description of the module and list keywords diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 446cf6dbb8..239a069397 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -13,7 +13,7 @@ def meta_yml(module_lint_object, module): The lint test checks that the module has a ``meta.yml`` file and that it follows the - JSON schema defined in the ``modules/yaml-schema.json`` + JSON schema defined in the ``modules/meta-schema.json`` file in the nf-core/modules repository. In addition it checks that the module name @@ -45,7 +45,7 @@ def meta_yml(module_lint_object, module): # Confirm that the meta.yml file is valid according to the JSON schema valid_meta_yml = True try: - with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/yaml-schema.json"), "r") as fh: + with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/meta-schema.json"), "r") as fh: schema = json.load(fh) jsonschema.validators.validate(instance=meta_yaml, schema=schema) module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index c32657de7a..29b70e4f9a 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml index f93b5ee519..b92870831c 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: From aaa59453e3ec3f4d4a910dbc364e85ac349d6ae1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 17:01:22 +0100 Subject: [PATCH 057/158] add types --- nf_core/modules/lint/meta_yml.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 239a069397..4dfc489f52 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -1,13 +1,15 @@ import json from pathlib import Path -import jsonschema.validators import yaml +from jsonschema import exceptions, validators +from nf_core.components.lint import ComponentLint +from nf_core.components.nfcore_component import NFCoreComponent from nf_core.modules.modules_differ import ModulesDiffer -def meta_yml(module_lint_object, module): +def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: """ Lint a ``meta.yml`` file @@ -47,15 +49,15 @@ def meta_yml(module_lint_object, module): try: with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/meta-schema.json"), "r") as fh: schema = json.load(fh) - jsonschema.validators.validate(instance=meta_yaml, schema=schema) + validators.validate(instance=meta_yaml, schema=schema) module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) - except jsonschema.exceptions.ValidationError as e: + except exceptions.ValidationError as e: valid_meta_yml = False hint = "" if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." if e.message.startswith("None is not of type 'object'") and len(e.path) > 2: - hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." + hint = f"\nCheck that the child entries of {str(e.path[0])+'.'+str(e.path[2])} are indented correctly." module.failed.append( ( "meta_yml_valid", @@ -63,7 +65,6 @@ def meta_yml(module_lint_object, module): module.meta_yml, ) ) - return # Confirm that all input and output channels are specified if valid_meta_yml: From 9fe1c3e66c06fa44881c8a68450f38d9b06e2717 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 17:24:02 +0100 Subject: [PATCH 058/158] add schema based validation to environment yaml linting, add tests --- nf_core/components/create.py | 1 - nf_core/components/lint/__init__.py | 3 +- nf_core/components/nfcore_component.py | 2 +- nf_core/modules/lint/environment_yml.py | 69 ++++++++++++++----------- nf_core/utils.py | 3 ++ tests/modules/lint.py | 68 ++++++++++++++++++++++-- tests/test_modules.py | 25 +++++---- tests/utils.py | 2 +- 8 files changed, 124 insertions(+), 49 deletions(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 42e769a340..224b701d77 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -49,7 +49,6 @@ def __init__( self.subtool = None self.tool_conda_name = conda_name self.tool_conda_version = conda_version - self.tool_licence = None self.tool_licence = "" self.tool_description = "" self.tool_doc_url = "" diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index ed0730d7b4..c46d68ff22 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -144,6 +144,7 @@ def __init__( def get_all_module_lint_tests(is_pipeline): if is_pipeline: return [ + "environment_yml", "module_patch", "module_version", "main_nf", @@ -153,7 +154,7 @@ def get_all_module_lint_tests(is_pipeline): "module_changes", ] else: - return ["main_nf", "meta_yml", "module_todos", "module_deprecations", "module_tests"] + return ["environment_yml", "main_nf", "meta_yml", "module_todos", "module_deprecations", "module_tests"] @staticmethod def get_all_subworkflow_lint_tests(is_pipeline): diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 0295d7c90d..fa306977fa 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -66,7 +66,7 @@ def __init__( self.component_name = self.component_dir.stem # These attributes are only used by nf-core modules # so just initialize them to None - self.meta_yml = None + self.meta_yml = "" self.test_dir = None self.test_yml = None self.test_main_nf = None diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index e475ef0e46..80c4132c13 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -2,12 +2,13 @@ from pathlib import Path import yaml +from jsonschema import exceptions, validators from nf_core.components.lint import ComponentLint from nf_core.components.nfcore_component import NFCoreComponent -def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent): +def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: """ Lint an ``environment.yml`` file. @@ -27,41 +28,49 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent): module.failed.append( ("environment_yml_exists", "Module `environment.yml` does not exist", module.environment_yml) ) - return - # Check that the dependencies section is sorted alphabetically + # Confirm that the environment.yml file is valid according to the JSON schema if env_yml: - if "dependencies" in env_yml: - if isinstance(env_yml["dependencies"], list): - if sorted(env_yml["dependencies"]) == env_yml["dependencies"]: - module.passed.append( - ( - "environment_yml_sorted", - "Module's `environment.yml` is sorted alphabetically", - module.environment_yml, - ) - ) - else: - module.failed.append( - ( - "environment_yml_sorted", - "Module's `environment.yml` is not sorted alphabetically", - module.environment_yml, - ) + valid_env_yml = False + try: + with open( + Path(module_lint_object.modules_repo.local_repo_dir, "modules/environment-schema.json"), "r" + ) as fh: + schema = json.load(fh) + validators.validate(instance=env_yml, schema=schema) + module.passed.append( + ("environment_yml_valid", "Module's `environment.yml` is valid", module.environment_yml) + ) + valid_env_yml = True + except exceptions.ValidationError as e: + hint = "" + if len(e.path) > 0: + hint = f"\nCheck the entry for `{e.path[0]}`." + if e.schema.get("message"): + e.message = e.schema["message"] + module.failed.append( + ( + "environment_yml_valid", + f"The `environment.yml` of the module {module.component_name} is not valid: {e.message}.{hint}", + module.environment_yml, + ) + ) + + # Check that the dependencies section is sorted alphabetically + if valid_env_yml: + if sorted(env_yml["dependencies"]) == env_yml["dependencies"]: + module.passed.append( + ( + "environment_yml_sorted", + "Module's `environment.yml` is sorted alphabetically", + module.environment_yml, ) + ) else: module.failed.append( ( - "environment_yml_valid", - "Module's `environment.yml` doesn't have a correctly formatted `dependencies` section, expecting an array", + "environment_yml_sorted", + "Module's `environment.yml` is not sorted alphabetically", module.environment_yml, ) ) - else: - module.failed.append( - ( - "environment_yml_valid", - "Module's `environment.yml` doesn't contain the required `dependencies` section", - module.environment_yml, - ) - ) diff --git a/nf_core/utils.py b/nf_core/utils.py index ee341822a9..8109a071ed 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -677,6 +677,9 @@ def parse_anaconda_licence(anaconda_response, version=None): l = l.replace("Clause", "clause") # BSD capitilisation l = re.sub(r"-only$", "", l) # Remove superflous GPL "only" version suffixes clean_licences.append(l) + # convert to string if only one licence + if len(clean_licences) == 1: + clean_licences = clean_licences[0] return clean_licences diff --git a/tests/modules/lint.py b/tests/modules/lint.py index c5d85c4b65..46be16c04c 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,7 +1,7 @@ -import os from pathlib import Path import pytest +import yaml import nf_core.modules from nf_core.modules.lint import main_nf @@ -10,7 +10,7 @@ from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf -def setup_patch(pipeline_dir, modify_module): +def setup_patch(pipeline_dir: str, modify_module: bool): install_obj = nf_core.modules.ModuleInstall( pipeline_dir, prompt=False, force=False, remote_url=GITLAB_URL, branch=PATCH_BRANCH, sha=CORRECT_SHA ) @@ -310,7 +310,6 @@ def test_modules_lint_check_url(self): def test_modules_lint_snapshot_file(self): """Test linting a module with a snapshot file""" - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" @@ -320,11 +319,14 @@ def test_modules_lint_snapshot_file(self): def test_modules_lint_snapshot_file_missing_fail(self): """Test linting a module with a snapshot file missing, which should fail""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").unlink() module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) - module_lint.lint(print_results=False, module="bpipe/test") + module_lint.lint(print_results=False, module="fastqc") + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "snapshot_file" def test_modules_lint_snapshot_file_not_needed(self): @@ -339,3 +341,61 @@ def test_modules_lint_snapshot_file_not_needed(self): assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 + + +def test_modules_environment_yml_file_doesnt_exists(self): + """Test linting a module with an environment.yml file""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml.bak") + ) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml.bak").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") + ) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "environment_yml_exists" + + +def test_modules_environment_yml_file_sorted_correctly(self): + """Test linting a module with a correctly sorted environment.yml file""" + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + + +def test_modules_environment_yml_file_sorted_incorrectly(self): + """Test linting a module with an incorrectly sorted environment.yml file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "r") as fh: + yaml_content = yaml.safe_load(fh) + # Add a new dependency to the environment.yml file and reverse the order + yaml_content["dependencies"].append("z") + yaml_content["dependencies"].reverse() + yaml_content = yaml.dump(yaml_content) + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "environment_yml_sorted" + + +def test_modules_environment_yml_file_not_array(self): + """Test linting a module with an incorrectly formatted environment.yml file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml")) as fh: + yaml_content = yaml.safe_load(fh) + yaml_content["dependencies"] = "z" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml.dump(yaml_content)) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "environment_yml_valid" diff --git a/tests/test_modules.py b/tests/test_modules.py index 4137163ed5..6031b4dfc5 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -5,6 +5,7 @@ import shutil import tempfile import unittest +from pathlib import Path import requests_cache import responses @@ -27,15 +28,12 @@ def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" - root_dir = os.path.join(tmp_dir, "modules") - os.makedirs(os.path.join(root_dir, "modules", "nf-core")) - os.makedirs(os.path.join(root_dir, "tests", "modules", "nf-core")) - os.makedirs(os.path.join(root_dir, "tests", "config")) - with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: - fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + root_dir = Path(tmp_dir, "modules") + Path(root_dir, "modules", "nf-core").mkdir(parents=True) + Path(root_dir, "tests", "modules", "nf-core").mkdir(parents=True) + Path(root_dir, "tests", "config").mkdir(parents=True) + with open(Path(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # mock biocontainers and anaconda response with responses.RequestsMock() as rsps: mock_anaconda_api_calls(rsps, "bpipe", "0.9.11--hdfd78af_0") @@ -46,7 +44,8 @@ def create_modules_repo_dummy(tmp_dir): module_create.create() # Remove doi from meta.yml which makes lint fail - meta_yml = os.path.join(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + meta_yml = Path(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() with open(meta_yml, "r") as fh: lines = fh.readlines() for line_index in range(len(lines)): @@ -69,9 +68,9 @@ def setUp(self): # Set up the schema root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") + self.template_dir = Path(root_repo_dir, "nf_core", "pipeline-template") self.pipeline_name = "mypipeline" - self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) + self.pipeline_dir = Path(self.tmp_dir, self.pipeline_name) nf_core.create.PipelineCreate( self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True ).init_pipeline() @@ -166,6 +165,10 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( # type: ignore[misc] + test_modules_environment_yml_file_doesnt_exists, + test_modules_environment_yml_file_not_array, + test_modules_environment_yml_file_sorted_correctly, + test_modules_environment_yml_file_sorted_incorrectly, test_modules_lint_check_process_labels, test_modules_lint_check_url, test_modules_lint_empty, diff --git a/tests/utils.py b/tests/utils.py index 5650d6d2eb..5c26485bb1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -84,7 +84,7 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): "doc_url": "http://test", "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], - "license": "", + "license": "MIT", } rsps.get(anaconda_api_url, json=anaconda_mock, status=200) From 2d51ff8fc3a08658e7ee6fd35b9721b9e83766fe Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 17:34:29 +0100 Subject: [PATCH 059/158] fix mypy --- nf_core/components/nfcore_component.py | 2 ++ nf_core/modules/lint/__init__.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index fa306977fa..6168d6ad28 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -46,6 +46,7 @@ def __init__( # Initialize the important files self.main_nf = self.component_dir / "main.nf" self.meta_yml = self.component_dir / "meta.yml" + self.process_name = "" self.environment_yml = self.component_dir / "environment.yml" repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1] @@ -67,6 +68,7 @@ def __init__( # These attributes are only used by nf-core modules # so just initialize them to None self.meta_yml = "" + self.environment_yml = "" self.test_dir = None self.test_yml = None self.test_main_nf = None diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 22f222c597..68a38cc0cd 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -29,15 +29,15 @@ class ModuleLint(ComponentLint): """ # Import lint functions - from .environment_yml import environment_yml - from .main_nf import main_nf - from .meta_yml import meta_yml - from .module_changes import module_changes - from .module_deprecations import module_deprecations - from .module_patch import module_patch - from .module_tests import module_tests - from .module_todos import module_todos - from .module_version import module_version + from .environment_yml import environment_yml # type: ignore[misc] + from .main_nf import main_nf # type: ignore[misc] + from .meta_yml import meta_yml # type: ignore[misc] + from .module_changes import module_changes # type: ignore[misc] + from .module_deprecations import module_deprecations # type: ignore[misc] + from .module_patch import module_patch # type: ignore[misc] + from .module_tests import module_tests # type: ignore[misc] + from .module_todos import module_todos # type: ignore[misc] + from .module_version import module_version # type: ignore[misc] def __init__( self, From 54bb5b6af01ad4326d5933332eac9c2618d254ed Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 17:38:01 +0100 Subject: [PATCH 060/158] fix type error --- nf_core/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index 8109a071ed..ea3351cf1e 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -17,7 +17,7 @@ import sys import time from pathlib import Path -from typing import Union +from typing import Tuple, Union import git import prompt_toolkit @@ -306,7 +306,7 @@ def fetch_wf_config(wf_path, cache_config=True): return config -def run_cmd(executable: str, cmd: str) -> Union[tuple[bytes, bytes], None]: +def run_cmd(executable: str, cmd: str) -> Union[Tuple[bytes, bytes], None]: """Run a specified command and capture the output. Handle errors nicely.""" full_cmd = f"{executable} {cmd}" log.debug(f"Running command: {full_cmd}") From 28c1c7657c9da71ef9c0d49b8c095e384abfbb23 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 19:57:57 +0100 Subject: [PATCH 061/158] add nf-test dependency --- nf_core/utils.py | 3 --- requirements-dev.txt | 1 + requirements.txt | 2 ++ 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/utils.py b/nf_core/utils.py index ea3351cf1e..9326ff1c93 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -677,9 +677,6 @@ def parse_anaconda_licence(anaconda_response, version=None): l = l.replace("Clause", "clause") # BSD capitilisation l = re.sub(r"-only$", "", l) # Remove superflous GPL "only" version suffixes clean_licences.append(l) - # convert to string if only one licence - if len(clean_licences) == 1: - clean_licences = clean_licences[0] return clean_licences diff --git a/requirements-dev.txt b/requirements-dev.txt index c94874b193..fd80c3fa0b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ black isort myst_parser +nf-test pytest-cov pytest-datafiles responses diff --git a/requirements.txt b/requirements.txt index 9cc7fc6be5..9b2a8dab92 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ GitPython jinja2 jsonschema>=3.0 markdown>=3.3 +nf-test packaging pre-commit prompt_toolkit>=3.0.3 @@ -17,3 +18,4 @@ requests_cache rich-click>=1.6.1 rich>=13.3.1 tabulate + From dc7a330d8fc6df7c47fff00ad80a9d9e9006aca5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 13 Nov 2023 20:09:32 +0100 Subject: [PATCH 062/158] install nf-test for ci --- .github/workflows/lint-code.yml | 9 +++++++++ Dockerfile | 5 +++++ requirements-dev.txt | 1 - requirements.txt | 1 - 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index c0de5c719d..38ddabb586 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -100,6 +100,15 @@ jobs: python -m pip install --upgrade pip -r requirements-dev.txt pip install -e . + - name: Setup conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.11 + + - name: Install nf-test + run: | + conda install -c bioconda nf-test + - name: Get Python changed files id: changed-py-files uses: tj-actions/changed-files@v23 diff --git a/Dockerfile b/Dockerfile index b148c4b544..95d544b26f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,11 @@ ENV NXF_VER ${NXF_VER} RUN curl -s https://get.nextflow.io | bash \ && mv nextflow /usr/local/bin \ && chmod a+rx /usr/local/bin/nextflow +# Install nf-test +RUN curl -fsSL https://code.askimed.com/install/nf-test | bash \ + && mv nf-test /usr/local/bin \ + && chmod a+rx /usr/local/bin/nf-test + # Add the nf-core source files to the image COPY . /usr/src/nf_core WORKDIR /usr/src/nf_core diff --git a/requirements-dev.txt b/requirements-dev.txt index fd80c3fa0b..c94874b193 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,6 @@ black isort myst_parser -nf-test pytest-cov pytest-datafiles responses diff --git a/requirements.txt b/requirements.txt index 9b2a8dab92..a4aaae25e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ GitPython jinja2 jsonschema>=3.0 markdown>=3.3 -nf-test packaging pre-commit prompt_toolkit>=3.0.3 From 1ce81c0178c595d9d0a586561c303166108f32c8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:47:33 +0100 Subject: [PATCH 063/158] switch bump_version to use environemt.yml --- nf_core/modules/bump_versions.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index e8c5060c89..4936eb4114 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -9,9 +9,11 @@ import logging import os import re +from typing import List, Union import questionary import rich +import yaml from rich.console import Console from rich.markdown import Markdown from rich.table import Table @@ -19,6 +21,7 @@ import nf_core.modules.modules_utils import nf_core.utils from nf_core.components.components_command import ComponentCommand +from nf_core.components.nfcore_component import NFCoreComponent from nf_core.utils import plural_s as _s from nf_core.utils import rich_force_colors @@ -31,11 +34,13 @@ def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): self.up_to_date = None self.updated = None - self.failed = None + self.failed = [] self.show_up_to_date = None self.tools_config = {} - def bump_versions(self, module=None, all_modules=False, show_uptodate=False): + def bump_versions( + self, module: Union[NFCoreComponent, None] = None, all_modules: bool = False, show_uptodate: bool = False + ) -> None: """ Bump the container and conda version of single module or all modules @@ -116,7 +121,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self._print_results() - def bump_module_version(self, module): + def bump_module_version(self, module: NFCoreComponent) -> bool: """ Bump the bioconda and container version of a single NFCoreComponent @@ -225,19 +230,18 @@ def bump_module_version(self, module): self.up_to_date.append((f"Module version up to date: {module.component_name}", module.component_name)) return True - def get_bioconda_version(self, module): + def get_bioconda_version(self, module: NFCoreComponent) -> List[str]: """ Extract the bioconda version from a module """ # Check whether file exists and load it - bioconda_packages = False + bioconda_packages = [] try: - with open(module.main_nf, "r") as fh: - for l in fh: - if "bioconda::" in l: - bioconda_packages = [b for b in l.split() if "bioconda::" in b] + with open(module.environment_yml, "r") as fh: + env_yml = yaml.safe_load(fh) + bioconda_packages = env_yml.get("dependencies", []) except FileNotFoundError: - log.error(f"Could not read `main.nf` of {module.component_name} module.") + log.error(f"Could not read `environment.yml` of {module.component_name} module.") return bioconda_packages From 32ef61bc313d2e5518d3054ca65a1e5c5df4cad2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:48:33 +0100 Subject: [PATCH 064/158] update modules in pipeline template --- nf_core/pipeline-template/modules.json | 6 +-- .../dumpsoftwareversions/environment.yml | 7 ++++ .../custom/dumpsoftwareversions/main.nf | 6 +-- .../custom/dumpsoftwareversions/meta.yml | 7 ++-- .../dumpsoftwareversions/tests/main.nf.test | 38 +++++++++++++++++ .../tests/main.nf.test.snap | 27 ++++++++++++ .../dumpsoftwareversions/tests/tags.yml | 2 + .../modules/nf-core/fastqc/environment.yml | 7 ++++ .../modules/nf-core/fastqc/main.nf | 6 +-- .../modules/nf-core/fastqc/meta.yml | 5 +++ .../modules/nf-core/fastqc/tests/main.nf.test | 41 +++++++++++++++++++ .../nf-core/fastqc/tests/main.nf.test.snap | 10 +++++ .../modules/nf-core/fastqc/tests/tags.yml | 2 + .../modules/nf-core/multiqc/environment.yml | 7 ++++ .../modules/nf-core/multiqc/main.nf | 6 +-- .../modules/nf-core/multiqc/meta.yml | 9 ++-- 16 files changed, 168 insertions(+), 18 deletions(-) create mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml create mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test create mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap create mode 100644 nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap create mode 100644 nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml create mode 100644 nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 2154873150..d711c0facf 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,17 +7,17 @@ "nf-core": { "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", "installed_by": ["modules"] }, "fastqc": { "branch": "master", - "git_sha": "bd8092b67b5103bdd52e300f75889442275c3117", + "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", "installed_by": ["modules"] }, "multiqc": { "branch": "master", - "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", + "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", "installed_by": ["modules"] } } diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml new file mode 100644 index 0000000000..f0c63f6984 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/environment.yml @@ -0,0 +1,7 @@ +name: custom_dumpsoftwareversions +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.17 diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf index ebc8727339..7685b33cde 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -2,10 +2,10 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "bioconda::multiqc=1.14" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : + 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" input: path versions diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml index 29b70e4f9a..9414c32d71 100644 --- a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: custom_dumpsoftwareversions description: Custom module used to dump software versions within the nf-core pipeline template keywords: @@ -16,7 +16,6 @@ input: type: file description: YML file containing software versions pattern: "*.yml" - output: - yml: type: file @@ -30,7 +29,9 @@ output: type: file description: File containing software versions pattern: "versions.yml" - authors: - "@drpatelh" - "@grst" +maintainers: + - "@drpatelh" + - "@grst" diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test new file mode 100644 index 0000000000..eec1db10a2 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test @@ -0,0 +1,38 @@ +nextflow_process { + + name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" + script "../main.nf" + process "CUSTOM_DUMPSOFTWAREVERSIONS" + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "dumpsoftwareversions" + tag "custom/dumpsoftwareversions" + + test("Should run without failures") { + when { + process { + """ + def tool1_version = ''' + TOOL1: + tool1: 0.11.9 + '''.stripIndent() + + def tool2_version = ''' + TOOL2: + tool2: 1.9 + '''.stripIndent() + + input[0] = Channel.of(tool1_version, tool2_version).collectFile() + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap new file mode 100644 index 0000000000..4274ed57aa --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap @@ -0,0 +1,27 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" + ], + "1": [ + "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" + ], + "2": [ + "versions.yml:md5,3843ac526e762117eedf8825b40683df" + ], + "mqc_yml": [ + "software_versions_mqc.yml:md5,2570f4ba271ad08357b0d3d32a9cf84d" + ], + "versions": [ + "versions.yml:md5,3843ac526e762117eedf8825b40683df" + ], + "yml": [ + "software_versions.yml:md5,1c851188476409cda5752ce971b20b58" + ] + } + ], + "timestamp": "2023-11-03T14:43:22.157011" + } +} diff --git a/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml new file mode 100644 index 0000000000..405aa24ae3 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml @@ -0,0 +1,2 @@ +custom/dumpsoftwareversions: + - modules/nf-core/custom/dumpsoftwareversions/** diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml new file mode 100644 index 0000000000..1787b38a9a --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/environment.yml @@ -0,0 +1,7 @@ +name: fastqc +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::fastqc=0.12.1 diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 249f90644d..50e59f2b8c 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -2,10 +2,10 @@ process FASTQC { tag "$meta.id" label 'process_medium' - conda "bioconda::fastqc=0.11.9" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : - 'biocontainers/fastqc:0.11.9--0' }" + 'https://depot.galaxyproject.org/singularity/fastqc:0.12.1--hdfd78af_0' : + 'biocontainers/fastqc:0.12.1--hdfd78af_0' }" input: tuple val(meta), path(reads) diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml index 4da5bb5a06..ee5507e06b 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/meta.yml @@ -50,3 +50,8 @@ authors: - "@grst" - "@ewels" - "@FelixKrueger" +maintainers: + - "@drpatelh" + - "@grst" + - "@ewels" + - "@FelixKrueger" diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test new file mode 100644 index 0000000000..6437a144d9 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test @@ -0,0 +1,41 @@ +nextflow_process { + + name "Test Process FASTQC" + script "../main.nf" + process "FASTQC" + tag "modules" + tag "modules_nfcore" + tag "fastqc" + + test("Single-Read") { + + when { + params { + outdir = "$outputDir" + } + process { + """ + input[0] = [ + [ id: 'test', single_end:true ], + [ + file(params.test_data['sarscov2']['illumina']['test_1_fastq_gz'], checkIfExists: true) + ] + ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + // NOTE The report contains the date inside it, which means that the md5sum is stable per day, but not longer than that. So you can't md5sum it. + // looks like this:
Mon 2 Oct 2023
test.gz
+ // https://github.com/nf-core/modules/pull/3903#issuecomment-1743620039 + { assert process.out.html.get(0).get(1) ==~ ".*/test_fastqc.html" }, + { assert path(process.out.html.get(0).get(1)).getText().contains("File typeConventional base calls") }, + { assert snapshot(process.out.versions).match("versions") }, + { assert process.out.zip.get(0).get(1) ==~ ".*/test_fastqc.zip" } + ) + } + } +} diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap new file mode 100644 index 0000000000..636a32cead --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/main.nf.test.snap @@ -0,0 +1,10 @@ +{ + "versions": { + "content": [ + [ + "versions.yml:md5,e1cc25ca8af856014824abd842e93978" + ] + ], + "timestamp": "2023-10-09T23:40:54+0000" + } +} \ No newline at end of file diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml new file mode 100644 index 0000000000..7834294ba0 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/tests/tags.yml @@ -0,0 +1,2 @@ +fastqc: + - modules/nf-core/fastqc/** diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml new file mode 100644 index 0000000000..e5e14e8395 --- /dev/null +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/environment.yml @@ -0,0 +1,7 @@ +name: MultiQC +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::multiqc=1.17 diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf index 1fc387beed..2bbc3983fa 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/main.nf @@ -1,10 +1,10 @@ process MULTIQC { label 'process_single' - conda "bioconda::multiqc=1.14" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.14--pyhdfd78af_0' : - 'biocontainers/multiqc:1.14--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.17--pyhdfd78af_0' : + 'biocontainers/multiqc:1.17--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml index b92870831c..a61223ed7a 100644 --- a/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/multiqc/meta.yml @@ -1,4 +1,4 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/yaml-schema.json name: MultiQC description: Aggregate results from bioinformatics analyses across many samples into a single report keywords: @@ -13,7 +13,6 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] - input: - multiqc_files: type: file @@ -31,7 +30,6 @@ input: type: file description: Optional logo file for MultiQC pattern: "*.{png}" - output: - report: type: file @@ -54,3 +52,8 @@ authors: - "@bunop" - "@drpatelh" - "@jfy133" +maintainers: + - "@abhi18av" + - "@bunop" + - "@drpatelh" + - "@jfy133" From ab113c7f0ec0793b040e249f3907fc7a971f4205 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:49:09 +0100 Subject: [PATCH 065/158] fix test for updated snapshot --- tests/components/create_snapshot.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index 9e2482baf2..588d97bbed 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -62,9 +62,16 @@ def test_generate_snapshot_subworkflow(self): def test_update_snapshot_module(self): """Update the snapshot of a module in nf-core/modules clone""" - original_timestamp = "2023-10-18T11:02:55.420631681" with set_wd(self.nfcore_modules): + snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") + with open(snap_path, "r") as fh: + snap_content = json.load(fh) + original_timestamp = snap_content["Single-End"]["timestamp"] + # delete the timestamp in json + snap_content["Single-End"]["content"][0]["0"][0][1] = "" + with open(snap_path, "w") as fh: + json.dump(snap_content, fh) snap_generator = ComponentTestSnapshotGenerator( component_type="modules", component_name="bwa/mem", @@ -75,9 +82,6 @@ def test_update_snapshot_module(self): ) snap_generator.run() - snap_path = Path("modules", "nf-core-test", "bwa", "mem", "tests", "main.nf.test.snap") - assert snap_path.exists() - with open(snap_path, "r") as fh: snap_content = json.load(fh) assert "Single-End" in snap_content From bec4bb35d9ce3860da029b68aee1062b88442f6f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:49:38 +0100 Subject: [PATCH 066/158] test for name mismatch in environment.yml --- nf_core/modules/lint/environment_yml.py | 19 ++++++++++++++++++- tests/modules/lint.py | 22 +++++++++++++++++++++- tests/test_modules.py | 1 + 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 80c4132c13..81509efb04 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -56,8 +56,8 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) ) ) - # Check that the dependencies section is sorted alphabetically if valid_env_yml: + # Check that the dependencies section is sorted alphabetically if sorted(env_yml["dependencies"]) == env_yml["dependencies"]: module.passed.append( ( @@ -74,3 +74,20 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.environment_yml, ) ) + # Check that the name in the environment.yml file matches the module name + if env_yml["name"] == module.component_name: + module.passed.append( + ( + "environment_yml_name", + "Module's `environment.yml` name matches module name", + module.environment_yml, + ) + ) + else: + module.failed.append( + ( + "environment_yml_name", + "Module's `environment.yml` name does not match module name", + module.environment_yml, + ) + ) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 46be16c04c..6bd6f9e659 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -394,8 +394,28 @@ def test_modules_environment_yml_file_not_array(self): with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: fh.write(yaml.dump(yaml_content)) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) - module_lint.lint(module="bpipe/test") + module_lint.lint(print_results=False, module="bpipe/test") assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "environment_yml_valid" + + +def test_modules_environment_yml_file_name_mismatch(self): + """Test linting a module with a different name in the environment.yml file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml")) as fh: + yaml_content = yaml.safe_load(fh) + yaml_content["name"] = "bpipe-test" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml.dump(yaml_content)) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(module="bpipe/test") + # reset changes + yaml_content["name"] = "bpipe_test" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml.dump(yaml_content)) + + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "environment_yml_name" diff --git a/tests/test_modules.py b/tests/test_modules.py index 6031b4dfc5..01c858c136 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -166,6 +166,7 @@ def test_modulesrepo_class(self): ) from .modules.lint import ( # type: ignore[misc] test_modules_environment_yml_file_doesnt_exists, + test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, test_modules_environment_yml_file_sorted_correctly, test_modules_environment_yml_file_sorted_incorrectly, From 0622fbc29223931515ecd5e1b2214afa115a835f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:51:35 +0100 Subject: [PATCH 067/158] make nicer diff for nf-test error --- nf_core/components/snapshot_generator.py | 28 +++++++++++++++++++++++- requirements.txt | 2 +- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index c0b1ae2805..9b73387963 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -55,6 +55,10 @@ def __init__( def run(self) -> None: """Run build steps""" self.check_inputs() + os.environ["NFT_DIFF"] = "icdiff" # set nf-test differ to icdiff to get a better diff output + os.environ[ + "NFT_DIFF_ARGS" + ] = "-N --cols 120 -L old_snapshot -L new_snapshot" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences with set_wd(self.dir): self.check_snapshot_stability() if len(self.errors) > 0: @@ -114,8 +118,30 @@ def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: nftest_output = Text.from_ansi(nftest_out.decode()) print(Panel(nftest_output, title="nf-test output")) if nftest_err: - syntax = Syntax(nftest_err.decode(), "java", theme="ansi_dark") + syntax = Syntax(nftest_err.decode(), "diff", theme="ansi_dark") print(Panel(syntax, title="nf-test error")) + if "Different Snapshot:" in nftest_err.decode(): + log.error("nf-test failed due to differences in the snapshots") + # prompt to update snapshot + if self.no_prompts: + log.info("Updating snapshot") + self.update = True + else: + answer = Confirm.ask( + "[bold][blue]?[/] nf-test found differences in the snapshot. Do you want to update it?", + default=True, + ) + if answer: + log.info("Updating snapshot") + self.update = True + else: + log.debug("Snapshot not updated") + if self.update: + # update snapshot using nf-test --update-snapshot + self.generate_snapshot() + + else: + self.errors.append("nf-test failed") def generate_snapshot(self) -> bool: """Generate the nf-test snapshot using `nf-test test` command diff --git a/requirements.txt b/requirements.txt index a4aaae25e2..ace9b44a5e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ requests_cache rich-click>=1.6.1 rich>=13.3.1 tabulate - +icdiff From 4c30da374836e1a6f11d4d7e34fdcff6578b7deb Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:51:49 +0100 Subject: [PATCH 068/158] fix error in download test --- tests/test_download.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 9416d462e4..feb2090b27 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -156,25 +156,27 @@ def test_find_container_images_config_basic(self, tmp_path, mock_fetch_wf_config @mock.patch("nf_core.utils.fetch_wf_config") def test__find_container_images_config_nextflow(self, tmp_path, mock_fetch_wf_config): download_obj = DownloadWorkflow(pipeline="dummy", outdir=tmp_path) - nfconfig_raw = run_cmd( - "nextflow", f"config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}" - ) - config = {} - for l in nfconfig_raw.splitlines(): - ul = l.decode("utf-8") - try: - k, v = ul.split(" = ", 1) - config[k] = v.strip("'\"") - except ValueError: - pass - mock_fetch_wf_config.return_value = config - download_obj.find_container_images("workflow") - assert len(download_obj.containers) == 4 - assert "nfcore/methylseq:1.0" in download_obj.containers - assert "nfcore/methylseq:1.4" in download_obj.containers - assert "nfcore/sarek:dev" in download_obj.containers - assert "https://depot.galaxyproject.org/singularity/r-shinyngs:1.7.1--r42hdfd78af_1" in download_obj.containers - # does not yet pick up nfcore/sarekvep:dev.${params.genome}, because that is no valid URL or Docker URI. + result = run_cmd("nextflow", f"config -flat {Path(__file__).resolve().parent / 'data/mock_config_containers'}") + if result is not None: + nfconfig_raw, _ = result + config = {} + for l in nfconfig_raw.splitlines(): + ul = l.decode("utf-8") + try: + k, v = ul.split(" = ", 1) + config[k] = v.strip("'\"") + except ValueError: + pass + mock_fetch_wf_config.return_value = config + download_obj.find_container_images("workflow") + assert len(download_obj.containers) == 4 + assert "nfcore/methylseq:1.0" in download_obj.containers + assert "nfcore/methylseq:1.4" in download_obj.containers + assert "nfcore/sarek:dev" in download_obj.containers + assert ( + "https://depot.galaxyproject.org/singularity/r-shinyngs:1.7.1--r42hdfd78af_1" in download_obj.containers + ) + # does not yet pick up nfcore/sarekvep:dev.${params.genome}, because that is no valid URL or Docker URI. # # Test for 'find_container_images' in modules From 2ca176432f3078856219bd031cbb477cff98e64f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 15:52:11 +0100 Subject: [PATCH 069/158] don't convert arrays to strings in licence field --- nf_core/module-template/modules/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index c27d21e563..a63acbb067 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -19,7 +19,7 @@ tools: documentation: "{{ tool_doc_url }}" tool_dev_url: "{{ tool_dev_url }}" doi: "" - licence: "{{ tool_licence }}" + licence: {{ tool_licence }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as input From 6e2dc70d906e712ff6fa73d773b9a0b542babeeb Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 16:09:20 +0100 Subject: [PATCH 070/158] add nf-test to CI --- .github/workflows/pytest.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 388aaeb3d7..655890125c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -64,6 +64,15 @@ jobs: with: version: "latest-everything" + - name: Setup conda + uses: conda-incubator/setup-miniconda@v2 + with: + python-version: 3.11 + + - name: Install nf-test + run: | + conda install -c bioconda nf-test + - name: Test with pytest run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core From cb175e7f8551adc29df040879e25389b42f8f8cc Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 17:02:06 +0100 Subject: [PATCH 071/158] fix environment.yml linting --- .../module-template/modules/environment.yml | 2 +- nf_core/modules/lint/environment_yml.py | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/nf_core/module-template/modules/environment.yml b/nf_core/module-template/modules/environment.yml index 61a36cd2e3..dcf510affb 100644 --- a/nf_core/module-template/modules/environment.yml +++ b/nf_core/module-template/modules/environment.yml @@ -1,6 +1,6 @@ --- # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json -name: "{{ component }}" +name: "{{ component_name_underscore }}" channels: - conda-forge - bioconda diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 81509efb04..9983ab1c31 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -22,12 +22,24 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) with open(Path(module.component_dir, "environment.yml"), "r") as fh: env_yml = yaml.safe_load(fh) - module.passed.append(("environment_yml_exists", "Module `environment.yml` exists", module.environment_yml)) + module.passed.append(("environment_yml_exists", "Module's `environment.yml` exists", module.environment_yml)) except FileNotFoundError: - module.failed.append( - ("environment_yml_exists", "Module `environment.yml` does not exist", module.environment_yml) - ) + # check if the module's main.nf requires a conda environment + with open(Path(module.component_dir, "main.nf"), "r") as fh: + main_nf = fh.read() + if "conda '${modulesDir}/environment.yml'" in main_nf: + module.failed.append( + ("environment_yml_exists", "Module's `environment.yml` does not exist", module.environment_yml) + ) + else: + module.passed.append( + ( + "environment_yml_exists", + "Module's `environment.yml` does not exist, but it is also not included in the main.nf", + module.environment_yml, + ) + ) # Confirm that the environment.yml file is valid according to the JSON schema if env_yml: @@ -74,8 +86,11 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.environment_yml, ) ) - # Check that the name in the environment.yml file matches the module name - if env_yml["name"] == module.component_name: + # Check that the name in the environment.yml file matches the name in the meta.yml file + with open(Path(module.component_dir, "meta.yml"), "r") as fh: + meta_yml = yaml.safe_load(fh) + + if env_yml["name"] == meta_yml["name"]: module.passed.append( ( "environment_yml_name", @@ -87,7 +102,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.failed.append( ( "environment_yml_name", - "Module's `environment.yml` name does not match module name", + f"Conflicting process name between environment.yml (`{env_yml['name']}`) and meta.yml (`{module.component_name}`)", module.environment_yml, ) ) From 38d9db73a480187e0035637c48154465ae104f77 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 17:03:00 +0100 Subject: [PATCH 072/158] rely on environment.yml for bioconda versions --- nf_core/modules/bump_versions.py | 8 +++++++- nf_core/modules/lint/main_nf.py | 10 +++++++++- tests/modules/bump_versions.py | 13 +++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4936eb4114..0df766a2c7 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -178,7 +178,6 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: return False patterns = [ - (bioconda_packages[0], f"'bioconda::{bioconda_tool_name}={last_ver}'"), (rf"biocontainers/{bioconda_tool_name}:[^'\"\s]+", docker_img), ( rf"https://depot.galaxyproject.org/singularity/{bioconda_tool_name}:[^'\"\s]+", @@ -218,6 +217,13 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: with open(module.main_nf, "w") as fh: fh.write(content) + # change version in environment.yml + with open(module.environment_yml, "r") as fh: + env_yml = yaml.safe_load(fh) + re.sub(bioconda_packages[0], f"'bioconda::{bioconda_tool_name}={last_ver}'", env_yml["dependencies"]) + with open(module.environment_yml, "w") as fh: + yaml.dump(env_yml, fh, default_flow_style=False) + self.updated.append( ( f"Module updated: {bioconda_version} --> {last_ver}", diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index d27797c7f2..13910f5f65 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -9,6 +9,7 @@ from urllib.parse import urlparse, urlunparse import requests +import yaml import nf_core import nf_core.modules.modules_utils @@ -255,7 +256,6 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): l = l.replace("container", "").strip(" \n'\"}:") if _container_type(l) == "conda": - bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) if match is None: self.passed.append( @@ -344,6 +344,14 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): ) ) + # Get bioconda packages from environment.yml + try: + with open(Path(self.component_dir, "environment.yml"), "r") as fh: + env_yml = yaml.safe_load(fh) + if "dependencies" in env_yml: + bioconda_packages = [x for x in env_yml["dependencies"] if isinstance(x, str) and "bioconda::" in x] + except FileNotFoundError: + pass # Check that all bioconda packages have build numbers # Also check for newer versions for bp in bioconda_packages: diff --git a/tests/modules/bump_versions.py b/tests/modules/bump_versions.py index 65569efd51..3c19041f63 100644 --- a/tests/modules/bump_versions.py +++ b/tests/modules/bump_versions.py @@ -2,6 +2,7 @@ import re import pytest +import yaml import nf_core.modules from nf_core.modules.modules_utils import ModuleException @@ -10,11 +11,11 @@ def test_modules_bump_versions_single_module(self): """Test updating a single module""" # Change the bpipe/test version to an older version - main_nf_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf") - with open(main_nf_path, "r") as fh: + env_yml_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") + with open(env_yml_path, "r") as fh: content = fh.read() new_content = re.sub(r"bioconda::star=\d.\d.\d\D?", r"bioconda::star=2.6.1d", content) - with open(main_nf_path, "w") as fh: + with open(env_yml_path, "w") as fh: fh.write(new_content) version_bumper = nf_core.modules.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) version_bumper.bump_versions(module="bpipe/test") @@ -39,11 +40,11 @@ def test_modules_bump_versions_fail(self): def test_modules_bump_versions_fail_unknown_version(self): """Fail because of an unknown version""" # Change the bpipe/test version to an older version - main_nf_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf") - with open(main_nf_path, "r") as fh: + env_yml_path = os.path.join(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml") + with open(env_yml_path, "r") as fh: content = fh.read() new_content = re.sub(r"bioconda::bpipe=\d.\d.\d\D?", r"bioconda::bpipe=xxx", content) - with open(main_nf_path, "w") as fh: + with open(env_yml_path, "w") as fh: fh.write(new_content) version_bumper = nf_core.modules.ModuleVersionBumper(pipeline_dir=self.nfcore_modules) version_bumper.bump_versions(module="bpipe/test") From 91c24e6c5ea9d80c19ea27723fdfcb93359c0434 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 17:03:12 +0100 Subject: [PATCH 073/158] fix tests --- nf_core/modules/lint/module_tests.py | 6 ++++-- nf_core/subworkflows/lint/subworkflow_tests.py | 6 ++++-- tests/modules/lint.py | 8 ++++---- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index bee01f6b5b..fd45843b7b 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -46,10 +46,12 @@ def module_tests(_, module): if "snapshot(" in fh.read(): snap_file = os.path.join(module.component_dir, "tests", "main.nf.test.snap") if os.path.exists(snap_file): - module.passed.append(("test_main_snap", "snapshot file `main.nf.test.snap` exists", snap_file)) + module.passed.append( + ("test_snapshot_exists", "snapshot file `main.nf.test.snap` exists", snap_file) + ) else: module.failed.append( - ("test_main_snap", "snapshot file `main.nf.test.snap` does not exist", snap_file) + ("test_snapshot_exists", "snapshot file `main.nf.test.snap` does not exist", snap_file) ) if os.path.exists(pytest_main_nf): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 4515dd21ca..4d3a670130 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -51,9 +51,11 @@ def subworkflow_tests(_, subworkflow): if "snapshot(" in fh.read(): snap_file = os.path.join(subworkflow.component_dir, "tests", "main.nf.test.snap") if os.path.exists(snap_file): - subworkflow.passed.append(("test_main_snap", "test `main.nf.test.snap` exists", snap_file)) + subworkflow.passed.append(("test_snapshot_exists", "test `main.nf.test.snap` exists", snap_file)) else: - subworkflow.failed.append(("test_main_snap", "test `main.nf.test.snap` does not exist", snap_file)) + subworkflow.failed.append( + ("test_snapshot_exists", "test `main.nf.test.snap` does not exist", snap_file) + ) if os.path.exists(pytest_main_nf): # Check that entry in pytest_modules.yml exists diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 6bd6f9e659..8263db1260 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -46,7 +46,7 @@ def test_modules_lint_empty(self): def test_modules_lint_new_modules(self): """lint a new module""" module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) - module_lint.lint(print_results=True, all_modules=True) + module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 @@ -321,12 +321,12 @@ def test_modules_lint_snapshot_file_missing_fail(self): """Test linting a module with a snapshot file missing, which should fail""" Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").unlink() module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) - module_lint.lint(print_results=False, module="fastqc") + module_lint.lint(print_results=False, module="bpipe/test") Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert module_lint.failed[0].lint_test == "snapshot_file" + assert module_lint.failed[0].lint_test == "test_snapshot_exists" def test_modules_lint_snapshot_file_not_needed(self): @@ -409,7 +409,7 @@ def test_modules_environment_yml_file_name_mismatch(self): with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: fh.write(yaml.dump(yaml_content)) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) - module_lint.lint(module="bpipe/test") + module_lint.lint(print_results=False, module="bpipe/test") # reset changes yaml_content["name"] = "bpipe_test" with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: From dc6d03385017586b35a0f81f77969a026bd6d3f2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 14 Nov 2023 17:41:32 +0100 Subject: [PATCH 074/158] fix types --- nf_core/modules/bump_versions.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 0df766a2c7..45bceba131 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -9,7 +9,7 @@ import logging import os import re -from typing import List, Union +from typing import Any, Dict, List, Optional, Tuple, Union import questionary import rich @@ -28,15 +28,22 @@ log = logging.getLogger(__name__) -class ModuleVersionBumper(ComponentCommand): - def __init__(self, pipeline_dir, remote_url=None, branch=None, no_pull=False): +class ModuleVersionBumper(ComponentCommand): # type: ignore[misc] + def __init__( + self, + pipeline_dir: str, + remote_url: Optional[str] = None, + branch: Optional[str] = None, + no_pull: bool = False, + ): super().__init__("modules", pipeline_dir, remote_url, branch, no_pull) - self.up_to_date = None - self.updated = None - self.failed = [] - self.show_up_to_date = None - self.tools_config = {} + self.up_to_date: List[Tuple[str, str]] = [] + self.updated: List[Tuple[str, str]] = [] + self.failed: List[Tuple[str, str]] = [] + self.ignored: List[Tuple[str, str]] = [] + self.show_up_to_date: Optional[bool] = None + self.tools_config: Dict[str, Any] = {} def bump_versions( self, module: Union[NFCoreComponent, None] = None, all_modules: bool = False, show_uptodate: bool = False @@ -251,7 +258,7 @@ def get_bioconda_version(self, module: NFCoreComponent) -> List[str]: return bioconda_packages - def _print_results(self): + def _print_results(self) -> None: """ Print the results for the bump_versions command Uses the ``rich`` library to print a set of formatted tables to the command line @@ -269,13 +276,13 @@ def _print_results(self): except: pass - def format_result(module_updates, table): + def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: """ Create rows for module updates """ # TODO: Row styles don't work current as table-level style overrides. # I'd like to make an issue about this on the rich repo so leaving here in case there is a future fix - last_modname = False + last_modname = "" row_style = None for module_update in module_updates: if last_modname and module_update[1] != last_modname: From d82bf6644c0486cad54592776052ef7ade70eeba Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 15 Nov 2023 10:10:48 +0100 Subject: [PATCH 075/158] remove subworkflows test_yml_builder and fix subworkflow tests --- nf_core/__main__.py | 2 +- nf_core/components/create.py | 7 +- nf_core/subworkflows/__init__.py | 1 - nf_core/subworkflows/test_yml_builder.py | 401 ----------------------- tests/subworkflows/create_test_yml.py | 96 ------ tests/subworkflows/lint.py | 20 +- tests/test_subworkflows.py | 7 - 7 files changed, 18 insertions(+), 516 deletions(-) delete mode 100644 nf_core/subworkflows/test_yml_builder.py delete mode 100644 tests/subworkflows/create_test_yml.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5de247e31e..0e2d6edb4e 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1088,7 +1088,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") -def create_test_yml(ctx, subworkflow, dir, run_tests, no_prompts, update): +def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update): """ Auto-generate a test.yml file for a new subworkflow. diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 224b701d77..54fade29a2 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -376,9 +376,10 @@ def _get_component_dirs(self): # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") - file_paths[os.path.join(self.component_type, "environment.yml")] = os.path.join( - component_dir, "environment.yml" - ) + if self.component_type == "modules": + file_paths[os.path.join(self.component_type, "environment.yml")] = os.path.join( + component_dir, "environment.yml" + ) file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( component_dir, "tests", "tags.yml" ) diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 1ceccd021f..825af5ecf5 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -5,5 +5,4 @@ from .list import SubworkflowList from .remove import SubworkflowRemove from .subworkflows_test import SubworkflowsTest -from .test_yml_builder import SubworkflowTestYmlBuilder from .update import SubworkflowUpdate diff --git a/nf_core/subworkflows/test_yml_builder.py b/nf_core/subworkflows/test_yml_builder.py deleted file mode 100644 index db239093da..0000000000 --- a/nf_core/subworkflows/test_yml_builder.py +++ /dev/null @@ -1,401 +0,0 @@ -""" -The ModulesTestYmlBuilder class handles automatic generation of the modules test.yml file -along with running the tests and creating md5 sums -""" - -from __future__ import print_function - -import errno -import gzip -import hashlib -import io -import logging -import operator -import os -import re -import shlex -import subprocess -import tempfile -from pathlib import Path - -import questionary -import rich -import yaml -from rich.syntax import Syntax - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson -from nf_core.modules.modules_repo import ModulesRepo - -from ..lint_utils import run_prettier_on_file - -log = logging.getLogger(__name__) - - -class SubworkflowTestYmlBuilder(ComponentCommand): - def __init__( - self, - subworkflow=None, - directory=".", - run_tests=False, - test_yml_output_path=None, - force_overwrite=False, - no_prompts=False, - remote_url=None, - branch=None, - ): - super().__init__("subworkflows", directory) - self.dir = directory - self.subworkflow = subworkflow - self.remote_url = remote_url - self.branch = branch - self.run_tests = run_tests - self.test_yml_output_path = test_yml_output_path - self.force_overwrite = force_overwrite - self.no_prompts = no_prompts - self.subworkflow_dir = None - self.subworkflow_test_main = None - self.entry_points = [] - self.tests = [] - self.errors = [] - self.modules_repo = ModulesRepo(remote_url=self.remote_url, branch=self.branch) - self.modules_json = ModulesJson(self.dir) - - def run(self): - """Run build steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self.check_inputs() - self.scrape_workflow_entry_points() - self.build_all_tests() - self.print_test_yml() - if len(self.errors) > 0: - errors = "\n - ".join(self.errors) - raise UserWarning(f"Ran, but found errors:\n - {errors}") - - def check_inputs(self): - """Do more complex checks about supplied flags.""" - # Get the tool name if not specified - if self.subworkflow is None: - self.subworkflow = questionary.autocomplete( - "Subworkflow name:", - choices=self.components_from_repo(self.org), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - self.subworkflow_dir = os.path.join("subworkflows", self.modules_repo.repo_path, self.subworkflow) - self.subworkflow_test_main = os.path.join( - "tests", "subworkflows", self.modules_repo.repo_path, self.subworkflow, "main.nf" - ) - - # First, sanity check that the module directory exists - if not os.path.isdir(self.subworkflow_dir): - raise UserWarning(f"Cannot find directory '{self.subworkflow_dir}'.") - if not os.path.exists(self.subworkflow_test_main): - raise UserWarning(f"Cannot find module test workflow '{self.subworkflow_test_main}'") - - # Check that we're running tests if no prompts - if not self.run_tests and self.no_prompts: - log.debug("Setting run_tests to True as running without prompts") - self.run_tests = True - - # Get the output YAML file / check it does not already exist - while self.test_yml_output_path is None: - default_val = f"tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow}/test.yml" - if self.no_prompts: - self.test_yml_output_path = default_val - else: - self.test_yml_output_path = rich.prompt.Prompt.ask( - "[violet]Test YAML output path[/] (- for stdout)", default=default_val - ).strip() - if self.test_yml_output_path == "": - self.test_yml_output_path = None - # Check that the output YAML file does not already exist - if ( - self.test_yml_output_path is not None - and self.test_yml_output_path != "-" - and os.path.exists(self.test_yml_output_path) - and not self.force_overwrite - ): - if rich.prompt.Confirm.ask( - f"[red]File exists! [green]'{self.test_yml_output_path}' [violet]Overwrite?" - ): - self.force_overwrite = True - else: - self.test_yml_output_path = None - if os.path.exists(self.test_yml_output_path) and not self.force_overwrite: - raise UserWarning( - f"Test YAML file already exists! '{self.test_yml_output_path}'. Use '--force' to overwrite." - ) - - def scrape_workflow_entry_points(self): - """Find the test workflow entry points from main.nf""" - log.info(f"Looking for test workflow entry points: '{self.subworkflow_test_main}'") - with open(self.subworkflow_test_main, "r") as fh: - for line in fh: - match = re.match(r"workflow\s+(\S+)\s+{", line) - if match: - self.entry_points.append(match.group(1)) - if len(self.entry_points) == 0: - raise UserWarning(f"No workflow entry points found in '{self.subworkflow_test_main}'") - - def build_all_tests(self): - """ - Go over each entry point and build structure - """ - - # Build the other tests - for entry_point in self.entry_points: - ep_test = self.build_single_test(entry_point) - if ep_test: - self.tests.append(ep_test) - - # Build the stub test - stub_test = self.build_single_test(self.entry_points[0], stub=True) - if stub_test: - self.tests.append(stub_test) - - def build_single_test(self, entry_point, stub=False): - """Given the supplied cli flags, prompt for any that are missing. - - Returns: Test command - """ - ep_test = { - "name": "", - "command": "", - "tags": [], - "files": [], - } - stub_option = " -stub" if stub else "" - stub_name = " stub" if stub else "" - - # Print nice divider line - console = rich.console.Console() - console.print("[black]" + "─" * console.width) - - log.info(f"Building test meta for entry point '{entry_point}'") - - while ep_test["name"] == "": - default_val = f"{self.subworkflow} {entry_point}{stub_name}" - if self.no_prompts: - ep_test["name"] = default_val - else: - ep_test["name"] = rich.prompt.Prompt.ask("[violet]Test name", default=default_val).strip() - - while ep_test["command"] == "": - default_val = f"nextflow run ./tests/subworkflows/{self.modules_repo.repo_path}/{self.subworkflow} -entry {entry_point} -c ./tests/config/nextflow.config{stub_option}" - if self.no_prompts: - ep_test["command"] = default_val - else: - ep_test["command"] = rich.prompt.Prompt.ask("[violet]Test command", default=default_val).strip() - - while len(ep_test["tags"]) == 0: - tag_defaults = ["subworkflows"] - tag_defaults.append("subworkflows/" + self.subworkflow) - tag_defaults += self.parse_module_tags(self.subworkflow_dir) - if self.no_prompts: - ep_test["tags"] = sorted(tag_defaults) - else: - while len(ep_test["tags"]) == 0: - prompt_tags = rich.prompt.Prompt.ask( - "[violet]Test tags[/] (comma separated)", default=",".join(sorted(tag_defaults)) - ).strip() - ep_test["tags"] = [t.strip() for t in prompt_tags.split(",")] - - ep_test["files"] = self.get_md5_sums(ep_test["command"], stub=stub) - - return ep_test - - def parse_module_tags(self, subworkflow_dir): - """ - Parse the subworkflow main.nf file to retrieve all imported modules for adding tags. - """ - tags = [] - with open(Path(subworkflow_dir, "main.nf"), "r") as fh: - for line in fh: - regex = re.compile( - r"include(?: *{ *)([a-zA-Z\_0-9]*)(?: *as *)?(?:[a-zA-Z\_0-9]*)?(?: *})(?: *from *)(?:'|\")(.*)(?:'|\")" - ) - match = regex.match(line) - if match and len(match.groups()) == 2: - name, link = match.groups() - if link.startswith("../../../"): - name_split = name.lower().split("_") - tags.append("/".join(name_split)) - if len(name_split) > 1: - tags.append(name_split[0]) - elif link.startswith("../"): - tags.append("subworkflows/" + name.lower()) - return list(set(tags)) - - def check_if_empty_file(self, fname): - """Check if the file is empty, or compressed empty""" - if os.path.getsize(fname) == 0: - return True - try: - with open(fname, "rb") as fh: - g_f = gzip.GzipFile(fileobj=fh, mode="rb") - if g_f.read() == b"": - return True - except gzip.BadGzipFile: - pass - - return False - - def _md5(self, fname): - """Generate md5 sum for file""" - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(io.DEFAULT_BUFFER_SIZE), b""): - hash_md5.update(chunk) - md5sum = hash_md5.hexdigest() - return md5sum - - def create_test_file_dict(self, results_dir, is_repeat=False, stub=False): - """Walk through directory and collect md5 sums""" - test_files = [] - for root, _, files in os.walk(results_dir, followlinks=True): - for filename in files: - # Check that the file is not versions.yml - if filename == "versions.yml": - continue - file_path = os.path.join(root, filename) - # add the key here so that it comes first in the dict - test_file = {"path": file_path} - # Check that this isn't an empty file - if self.check_if_empty_file(file_path) and not stub: - if not is_repeat: - self.errors.append(f"Empty file found! '{os.path.basename(file_path)}'") - # Add the md5 anyway, linting should fail later and can be manually removed if needed. - # Originally we skipped this if empty, but then it's too easy to miss the warning. - # Equally, if a file is legitimately empty we don't want to prevent this from working. - if not stub: - file_md5 = self._md5(file_path) - test_file["md5sum"] = file_md5 - # Switch out the results directory path with the expected 'output' directory - test_file["path"] = file_path.replace(results_dir, "output") - test_files.append(test_file) - - test_files = sorted(test_files, key=operator.itemgetter("path")) - - return test_files - - def get_md5_sums(self, command, results_dir=None, results_dir_repeat=None, stub=False): - """ - Recursively go through directories and subdirectories - and generate tuples of (, ) - returns: list of tuples - """ - - run_this_test = False - while results_dir is None: - if self.run_tests or run_this_test: - results_dir, results_dir_repeat = self.run_tests_workflow(command) - else: - results_dir = rich.prompt.Prompt.ask( - "[violet]Test output folder with results[/] (leave blank to run test)" - ) - if results_dir == "": - results_dir = None - run_this_test = True - elif not os.path.isdir(results_dir): - log.error(f"Directory '{results_dir}' does not exist") - results_dir = None - - test_files = self.create_test_file_dict(results_dir=results_dir, stub=stub) - - # If test was repeated, compare the md5 sums - if results_dir_repeat: - test_files_repeat = self.create_test_file_dict(results_dir=results_dir_repeat, is_repeat=True, stub=stub) - - # Compare both test.yml files - for i in range(len(test_files)): - if test_files[i].get("md5sum") and not test_files[i].get("md5sum") == test_files_repeat[i]["md5sum"]: - test_files[i].pop("md5sum") - test_files[i]["contains"] = [ - " # TODO nf-core: file md5sum was variable, please replace this text with a string found in the file instead " - ] - - if len(test_files) == 0: - raise UserWarning(f"Could not find any test result files in '{results_dir}'") - - return test_files - - def run_tests_workflow(self, command): - """Given a test workflow and an entry point, run the test workflow""" - - # The config expects $PROFILE and Nextflow fails if it's not set - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting env var '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose software profile", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - if profile in ["singularity", "conda"]: - os.environ["PROFILE"] = profile - log.info(f"Setting env var '$PROFILE' to '{profile}'") - - tmp_dir = tempfile.mkdtemp() - tmp_dir_repeat = tempfile.mkdtemp() - work_dir = tempfile.mkdtemp() - command_repeat = command + f" --outdir {tmp_dir_repeat} -work-dir {work_dir}" - command += f" --outdir {tmp_dir} -work-dir {work_dir}" - - log.info(f"Running '{self.subworkflow}' test with command:\n[violet]{command}") - try: - nfconfig_raw = subprocess.check_output(shlex.split(command)) - log.info("Repeating test ...") - nfconfig_raw = subprocess.check_output(shlex.split(command_repeat)) - - except OSError as e: - if e.errno == errno.ENOENT and command.strip().startswith("nextflow "): - raise AssertionError( - "It looks like Nextflow is not installed. It is required for most nf-core functions." - ) - except subprocess.CalledProcessError as e: - output = rich.markup.escape(e.output.decode()) - raise UserWarning(f"Error running test workflow (exit code {e.returncode})\n[red]{output}") - except Exception as e: - raise UserWarning(f"Error running test workflow: {e}") - else: - log.info("Test workflow finished!") - try: - log.debug(rich.markup.escape(nfconfig_raw)) - except TypeError: - log.debug(rich.markup.escape(nfconfig_raw.decode("utf-8"))) - - return tmp_dir, tmp_dir_repeat - - def print_test_yml(self): - """ - Generate the test yml file. - """ - with tempfile.NamedTemporaryFile(mode="w+") as fh: - yaml.dump(self.tests, fh, Dumper=nf_core.utils.custom_yaml_dumper(), width=10000000) - run_prettier_on_file(fh.name) - fh.seek(0) - prettified_yml = fh.read() - - if self.test_yml_output_path == "-": - console = rich.console.Console() - console.print("\n", Syntax(prettified_yml, "yaml"), "\n") - else: - try: - log.info(f"Writing to '{self.test_yml_output_path}'") - with open(self.test_yml_output_path, "w") as fh: - fh.write(prettified_yml) - except FileNotFoundError as e: - raise UserWarning(f"Could not create test.yml file: '{e}'") diff --git a/tests/subworkflows/create_test_yml.py b/tests/subworkflows/create_test_yml.py deleted file mode 100644 index 40384b420f..0000000000 --- a/tests/subworkflows/create_test_yml.py +++ /dev/null @@ -1,96 +0,0 @@ -import os -from pathlib import Path -from unittest import mock - -import pytest - -import nf_core.subworkflows - -from ..utils import with_temporary_folder - - -@with_temporary_folder -def test_subworkflows_custom_yml_dumper(self, out_dir): - """Try to create a yml file with the custom yml dumper""" - yml_output_path = Path(out_dir, "test.yml") - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( - subworkflow="test/tool", - directory=self.pipeline_dir, - test_yml_output_path=yml_output_path, - no_prompts=True, - ) - meta_builder.test_yml_output_path = yml_output_path - meta_builder.tests = [{"testname": "myname"}] - meta_builder.print_test_yml() - assert Path(yml_output_path).is_file() - - -@with_temporary_folder -def test_subworkflows_test_file_dict(self, test_file_dir): - """Create dict of test files and create md5 sums""" - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( - subworkflow="test/tool", - directory=self.pipeline_dir, - test_yml_output_path="./", - no_prompts=True, - ) - with open(Path(test_file_dir, "test_file.txt"), "w") as fh: - fh.write("this line is just for testing") - test_files = meta_builder.create_test_file_dict(test_file_dir) - assert len(test_files) == 1 - assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" - - -@with_temporary_folder -def test_subworkflows_create_test_yml_get_md5(self, test_file_dir): - """Get md5 sums from a dummy output""" - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( - subworkflow="test/tool", - directory=self.pipeline_dir, - test_yml_output_path="./", - no_prompts=True, - ) - with open(Path(test_file_dir, "test_file.txt"), "w") as fh: - fh.write("this line is just for testing") - test_files = meta_builder.get_md5_sums( - command="dummy", - results_dir=test_file_dir, - results_dir_repeat=test_file_dir, - ) - assert test_files[0]["md5sum"] == "2191e06b28b5ba82378bcc0672d01786" - - -def test_subworkflows_create_test_yml_entry_points(self): - """Test extracting test entry points from a main.nf file""" - subworkflow = "test_subworkflow" - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( - subworkflow=f"{subworkflow}/test", - directory=self.pipeline_dir, - test_yml_output_path="./", - no_prompts=True, - ) - meta_builder.subworkflow_test_main = Path( - self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" - ) - meta_builder.scrape_workflow_entry_points() - assert meta_builder.entry_points[0] == f"test_{subworkflow}" - - -def test_subworkflows_create_test_yml_check_inputs(self): - """Test the check_inputs() function - raise UserWarning because test.yml exists""" - cwd = os.getcwd() - os.chdir(self.nfcore_modules) - subworkflow = "test_subworkflow" - meta_builder = nf_core.subworkflows.SubworkflowTestYmlBuilder( - subworkflow=f"{subworkflow}", - directory=self.pipeline_dir, - test_yml_output_path="./", - no_prompts=True, - ) - meta_builder.subworkflow_test_main = Path( - self.nfcore_modules, "tests", "subworkflows", "nf-core", subworkflow, "main.nf" - ) - with pytest.raises(UserWarning) as excinfo: - meta_builder.check_inputs() - os.chdir(cwd) - assert "Test YAML file already exists!" in str(excinfo.value) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 36beda5029..5bbe746f2e 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -5,7 +5,7 @@ import nf_core.subworkflows -from ..utils import GITLAB_URL +from ..utils import GITLAB_SUBWORKFLOWS_BRANCH, GITLAB_URL def test_subworkflows_lint(self): @@ -28,7 +28,9 @@ def test_subworkflows_lint_new_subworkflow(self): """lint a new subworkflow""" subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) - assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert ( + len(subworkflow_lint.failed) == 1 # test snap missing after creating a subworkflow + ), f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -42,9 +44,11 @@ def test_subworkflows_lint_no_gitlab(self): def test_subworkflows_lint_gitlab_subworkflows(self): """Lint subworkflows from a different remote""" self.subworkflow_install_gitlab.install("bam_stats_samtools") - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) + subworkflow_lint = nf_core.subworkflows.SubworkflowLint( + dir=self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) subworkflow_lint.lint(print_results=False, all_subworkflows=True) - assert len(subworkflow_lint.failed) == 2 + assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -53,9 +57,11 @@ def test_subworkflows_lint_multiple_remotes(self): """Lint subworkflows from a different remote""" self.subworkflow_install_gitlab.install("bam_stats_samtools") self.subworkflow_install.install("fastq_align_bowtie2") - subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.pipeline_dir, remote_url=GITLAB_URL) - subworkflow_lint.lint(print_results=False, all_modules=True) - assert len(subworkflow_lint.failed) == 1 + subworkflow_lint = nf_core.subworkflows.SubworkflowLint( + dir=self.pipeline_dir, remote_url=GITLAB_URL, branch=GITLAB_SUBWORKFLOWS_BRANCH + ) + subworkflow_lint.lint(print_results=False, all_subworkflows=True) + assert len(subworkflow_lint.failed) == 0 assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index f64c900f10..b499848409 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -106,13 +106,6 @@ def tearDown(self): test_subworkflows_create_nfcore_modules, test_subworkflows_create_succeed, ) - from .subworkflows.create_test_yml import ( # type: ignore[misc] - test_subworkflows_create_test_yml_check_inputs, - test_subworkflows_create_test_yml_entry_points, - test_subworkflows_create_test_yml_get_md5, - test_subworkflows_custom_yml_dumper, - test_subworkflows_test_file_dict, - ) from .subworkflows.info import ( # type: ignore[misc] test_subworkflows_info_in_modules_repo, test_subworkflows_info_local, From 24ff6362d289a2d2a254b12471de1f0a783e75e1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 15 Nov 2023 11:33:25 +0100 Subject: [PATCH 076/158] fix nf-test setup in CI --- .github/workflows/lint-code.yml | 14 ++++++++++---- .github/workflows/pytest.yml | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 38ddabb586..edff9a8c82 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -100,14 +100,20 @@ jobs: python -m pip install --upgrade pip -r requirements-dev.txt pip install -e . - - name: Setup conda - uses: conda-incubator/setup-miniconda@v2 + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 with: - python-version: 3.11 + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' run: | - conda install -c bioconda nf-test + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ - name: Get Python changed files id: changed-py-files diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 655890125c..4d9881d6bb 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -64,14 +64,20 @@ jobs: with: version: "latest-everything" - - name: Setup conda - uses: conda-incubator/setup-miniconda@v2 + - name: Cache nf-test installation + id: cache-software + uses: actions/cache@v3 with: - python-version: 3.11 + path: | + /usr/local/bin/nf-test + /home/runner/.nf-test/nf-test.jar + key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest - name: Install nf-test + if: steps.cache-software.outputs.cache-hit != 'true' run: | - conda install -c bioconda nf-test + wget -qO- https://code.askimed.com/install/nf-test | bash + sudo mv nf-test /usr/local/bin/ - name: Test with pytest run: python3 -m pytest tests/ --color=yes --cov-report=xml --cov-config=.github/.coveragerc --cov=nf_core From 72a78c01f924a69c56665017e99751f329e6d0a9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 15 Nov 2023 16:38:10 +0100 Subject: [PATCH 077/158] fix assertion value --- tests/components/create_snapshot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index 588d97bbed..e8632536f1 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -100,7 +100,7 @@ def test_test_not_found(self): ) with pytest.raises(UserWarning) as e: snap_generator.run() - assert "Test file 'main.nf.test' not found" in e + assert "Test file 'main.nf.test' not found" in str(e.value) def test_unstable_snapshot(self): @@ -115,4 +115,4 @@ def test_unstable_snapshot(self): ) with pytest.raises(UserWarning) as e: snap_generator.run() - assert "nf-test snapshot is not stable" in e + assert "nf-test snapshot is not stable" in str(e.value) From d6874302d833f5f661a0de96d8d046ca0555169b Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 15 Nov 2023 16:51:48 +0100 Subject: [PATCH 078/158] handle local modules without nf-test --- nf_core/modules/lint/main_nf.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 13910f5f65..56a9e99925 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -256,6 +256,8 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): l = l.replace("container", "").strip(" \n'\"}:") if _container_type(l) == "conda": + if "bioconda::" in l: + bioconda_packages = [b for b in l.split() if "bioconda::" in b] match = re.search(r"params\.enable_conda", l) if match is None: self.passed.append( @@ -352,6 +354,9 @@ def check_process_section(self, lines, registry, fix_version, progress_bar): bioconda_packages = [x for x in env_yml["dependencies"] if isinstance(x, str) and "bioconda::" in x] except FileNotFoundError: pass + except NotADirectoryError: + pass + # Check that all bioconda packages have build numbers # Also check for newer versions for bp in bioconda_packages: From f673a89d63e9454afd74125f7620e78110624f02 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 15 Nov 2023 15:01:05 +0100 Subject: [PATCH 079/158] add linting for nf-test and remove linting for pytests --- nf_core/components/nfcore_component.py | 22 +- nf_core/modules/lint/module_tests.py | 166 ++++++++------ .../subworkflows/lint/subworkflow_tests.py | 205 +++++++++--------- 3 files changed, 214 insertions(+), 179 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 6168d6ad28..e59762e227 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -51,9 +51,9 @@ def __init__( repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1] self.org = repo_dir - self.test_dir = Path(self.base_dir, "tests", component_type, repo_dir, self.component_name) - self.test_yml = self.test_dir / "test.yml" - self.test_main_nf = self.test_dir / "main.nf" + self.nftest_testdir = self.component_dir / "tests" + self.nftest_main_nf = self.nftest_testdir / "main.nf.test" + self.tags_yml = self.nftest_testdir / "tags.yml" if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -72,3 +72,19 @@ def __init__( self.test_dir = None self.test_yml = None self.test_main_nf = None + + def _get_main_nf_tags(self, test_main_nf): + """Collect all tags from the main.nf.test file.""" + tags = [] + for line in test_main_nf: + if line.strip().startswith("tag"): + tags.append(line.strip().split()[1].strip('"')) + return tags + + def _get_included_components(self, main_nf): + """Collect all included components from the main.nf file.""" + included_components = [] + for line in main_nf: + if line.strip().startswith("include"): + included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) + return included_components diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index fd45843b7b..e03f6abc53 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -14,104 +14,128 @@ def module_tests(_, module): Lint the tests of a module in ``nf-core/modules`` It verifies that the test directory exists - and contains a ``main.nf`` and a ``test.yml``, - and that the module is present in the ``pytest_modules.yml`` - file. + and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. """ - nftest_testdir = os.path.join(module.component_dir, "tests") - if os.path.exists(nftest_testdir): - module.passed.append(("test_dir_exists", "nf-test test directory exists", nftest_testdir)) - elif os.path.exists(module.test_dir): - module.passed.append(("test_dir_exists", "Test directory exists", module.test_dir)) + if module.nftest_testdir.is_dir(): + module.passed.append(("test_dir_exists", "nf-test test directory exists", module.nftest_testdir)) else: - module.failed.append(("test_dir_exists", "Test directory is missing", module.test_dir)) - module.warned.append(("test_dir_exists", "nf-test directory is missing", nftest_testdir)) + module.failed.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) return # Lint the test main.nf file - pytest_main_nf = os.path.join(module.test_dir, "main.nf") - nftest_main_nf = os.path.join(module.component_dir, "tests", "main.nf.test") - if os.path.exists(nftest_main_nf): - module.passed.append(("test_main_exists", "test `main.nf.test` exists", nftest_main_nf)) - elif os.path.exists(pytest_main_nf): - module.passed.append(("test_main_exists", "test `main.nf` exists", module.test_main_nf)) + if module.nftest_main_nf.is_file(): + module.passed.append(("test_main_exists", "test `main.nf.test` exists", module.nftest_main_nf)) else: - module.failed.append(("test_main_exists", "test `main.nf` does not exist", module.test_main_nf)) - module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", nftest_main_nf)) + module.failed.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) - if os.path.exists(nftest_main_nf): + if module.nftest_main_nf.is_file(): # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test - with open(nftest_main_nf, "r") as fh: + with open(module.nftest_main_nf, "r") as fh: if "snapshot(" in fh.read(): - snap_file = os.path.join(module.component_dir, "tests", "main.nf.test.snap") - if os.path.exists(snap_file): + snap_file = module.nftest_testdir / "main.nf.test.snap" + if snap_file.is_file(): module.passed.append( ("test_snapshot_exists", "snapshot file `main.nf.test.snap` exists", snap_file) ) + # Validate no empty files + with open(snap_file, "r") as snap_fh: + snap_content = snap_fh.read() + if "d41d8cd98f00b204e9800998ecf8427e" in snap_content: + module.failed.append( + ( + "test_snap_md5sum", + "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e", + snap_file, + ) + ) + else: + module.passed.append( + ( + "test_snap_md5sum", + "no md5sum for empty file found", + snap_file, + ) + ) + if "7029066c27ac6f5ef18d660d5741979a" in snap_content: + module.failed.append( + ( + "test_snap_md5sum", + "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a", + snap_file, + ) + ) + else: + module.passed.append( + ( + "test_snap_md5sum", + "no md5sum for compressed empty file found", + snap_file, + ) + ) else: module.failed.append( ("test_snapshot_exists", "snapshot file `main.nf.test.snap` does not exist", snap_file) ) + # Verify that tags are correct. + main_nf_tags = module._get_main_nf_tags(fh) + required_tags = ["modules", "modules_nfcore", module.component_name] + if module.component_name.count("/") == 1: + required_tags.append(module.component_name.split("/")[0]) + missing_tags = [] + for tag in required_tags: + if tag not in main_nf_tags: + missing_tags.append(tag) + if len(missing_tags) == 0: + module.passed.append(("test_main_tags", "Tags adhere to guidelines", module.nftest_main_nf)) + else: + module.failed.append( + ( + "test_main_tags", + f"Tags do not adhere to guidelines. Tags missing in `main.nf.test`: {missing_tags}", + module.nftest_main_nf, + ) + ) - if os.path.exists(pytest_main_nf): - # Check that entry in pytest_modules.yml exists + # Check pytest_modules.yml does not contain entries for subworkflows with nf-test + pytest_yml_path = os.path.join(module.base_dir, "tests", "config", "pytest_modules.yml") + if pytest_yml_path.is_file(): try: - pytest_yml_path = os.path.join(module.base_dir, "tests", "config", "pytest_modules.yml") with open(pytest_yml_path, "r") as fh: pytest_yml = yaml.safe_load(fh) if module.component_name in pytest_yml.keys(): - module.passed.append(("test_pytest_yml", "correct entry in pytest_modules.yml", pytest_yml_path)) - elif os.path.exists(nftest_main_nf): - module.passed.append( + module.failed.append( ( "test_pytest_yml", - "missing entry in pytest_modules.yml, but found nf-test test", - nftest_main_nf, + "module with nf-test should not be listed in pytest_modules.yml", + pytest_yml_path, ) ) else: - module.failed.append(("test_pytest_yml", "missing entry in pytest_modules.yml", pytest_yml_path)) + module.passed.append( + ("test_pytest_yml", "module with nf-test not in pytest_modules.yml", pytest_yml_path) + ) except FileNotFoundError: - module.failed.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path)) - - # Lint the test.yml file - try: - with open(module.test_yml, "r") as fh: - test_yml = yaml.safe_load(fh) + module.warned.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path)) - # Verify that tags are correct - all_tags_correct = True - for test in test_yml: - for tag in test["tags"]: - if not tag in [module.component_name, module.component_name.split("/")[0]]: - all_tags_correct = False - - # Look for md5sums of empty files - for tfile in test.get("files", []): - if tfile.get("md5sum") == "d41d8cd98f00b204e9800998ecf8427e": - module.failed.append( - ( - "test_yml_md5sum", - "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e", - module.test_yml, - ) - ) - if tfile.get("md5sum") == "7029066c27ac6f5ef18d660d5741979a": - module.failed.append( - ( - "test_yml_md5sum", - "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a", - module.test_yml, - ) - ) - - if all_tags_correct: - module.passed.append(("test_yml_tags", "tags adhere to guidelines", module.test_yml)) + if module.tags_yml.is_file(): + # Check that tags.yml exists and it has the correct entry + module.passed.append(("test_tags_yml_exists", "file `tags.yml` exists", module.tags_yml)) + with open(module.tags_yml, "r") as fh: + tags_yml = yaml.safe_load(fh) + if module.component_name in tags_yml.keys(): + module.passed.append(("test_tags_yml", "correct entry in tags.yml", module.tags_yml)) + if f"modules/{module.org}/{module.component_name}/**" in tags_yml[module.component_name]: + module.passed.append(("test_tags_yml", "correct path in tags.yml", module.tags_yml)) else: - module.failed.append(("test_yml_tags", "tags do not adhere to guidelines", module.test_yml)) - - # Test that the file exists - module.passed.append(("test_yml_exists", "Test `test.yml` exists", module.test_yml)) - except FileNotFoundError: - module.failed.append(("test_yml_exists", "Test `test.yml` does not exist", module.test_yml)) + module.failed.append(("test_tags_yml", "incorrect path in tags.yml", module.tags_yml)) + else: + module.failed.append( + ( + "test_tags_yml", + "incorrect entry in tags.yml, should be '' or '/'", + module.tags_yml, + ) + ) + else: + module.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 4d3a670130..8a9429e151 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -17,143 +17,138 @@ def subworkflow_tests(_, subworkflow): Lint the tests of a subworkflow in ``nf-core/modules`` It verifies that the test directory exists - and contains a ``main.nf`` and a ``test.yml``, - and that the subworkflow is present in the ``pytest_modules.yml`` - file. + and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - - nftest_testdir = os.path.join(subworkflow.component_dir, "tests") - if os.path.exists(nftest_testdir): - subworkflow.passed.append(("test_dir_exists", "nf-test test directory exists", nftest_testdir)) - elif os.path.exists(subworkflow.test_dir): - subworkflow.passed.append(("test_dir_exists", "Test directory exists", subworkflow.test_dir)) + if subworkflow.nftest_testdir.is_dir(): + subworkflow.passed.append(("test_dir_exists", "nf-test test directory exists", subworkflow.nftest_testdir)) else: - subworkflow.failed.append(("test_dir_exists", "Test directory is missing", subworkflow.test_dir)) - subworkflow.warned.append(("test_dir_exists", "nf-test directory is missing", nftest_testdir)) + subworkflow.failed.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) return # Lint the test main.nf file - pytest_main_nf = os.path.join(subworkflow.test_dir, "main.nf") - nftest_main_nf = os.path.join(subworkflow.component_dir, "tests", "main.nf.test") - if os.path.exists(nftest_main_nf): - subworkflow.passed.append(("test_main_exists", "test `main.nf.test` exists", nftest_main_nf)) - elif os.path.exists(pytest_main_nf): - subworkflow.passed.append(("test_main_exists", "test `main.nf` exists", subworkflow.test_main_nf)) + if subworkflow.nftest_main_nf.is_file(): + subworkflow.passed.append(("test_main_exists", "test `main.nf.test` exists", subworkflow.nftest_main_nf)) else: - subworkflow.failed.append(("test_main_exists", "test `main.nf` does not exist", subworkflow.test_main_nf)) - subworkflow.warned.append(("test_main_exists", "test `main.nf.test` does not exist", nftest_main_nf)) + subworkflow.failed.append( + ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) + ) - if os.path.exists(nftest_main_nf): - # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test - with open(nftest_main_nf, "r") as fh: + if subworkflow.nftest_main_nf.is_file(): + with open(subworkflow.nftest_main_nf, "r") as fh: + # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test if "snapshot(" in fh.read(): - snap_file = os.path.join(subworkflow.component_dir, "tests", "main.nf.test.snap") - if os.path.exists(snap_file): + snap_file = subworkflow.nftest_testdir / "main.nf.test.snap" + if snap_file.is_file(): subworkflow.passed.append(("test_snapshot_exists", "test `main.nf.test.snap` exists", snap_file)) - else: - subworkflow.failed.append( - ("test_snapshot_exists", "test `main.nf.test.snap` does not exist", snap_file) - ) - - if os.path.exists(pytest_main_nf): - # Check that entry in pytest_modules.yml exists - try: - pytest_yml_path = os.path.join(subworkflow.base_dir, "tests", "config", "pytest_modules.yml") - with open(pytest_yml_path, "r") as fh: - pytest_yml = yaml.safe_load(fh) - if "subworkflows/" + subworkflow.component_name in pytest_yml.keys(): - subworkflow.passed.append( - ("test_pytest_yml", "correct entry in pytest_modules.yml", pytest_yml_path) - ) - elif os.path.exists(nftest_main_nf): - subworkflow.passed.append( - ( - "test_pytest_yml", - "missing entry in pytest_modules.yml, but found nf-test test", - nftest_main_nf, - ) - ) - else: - subworkflow.failed.append( - ("test_pytest_yml", "missing entry in pytest_modules.yml", pytest_yml_path) - ) - except FileNotFoundError: - subworkflow.failed.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path)) - - # Lint the test.yml file - try: - with open(subworkflow.test_yml, "r") as fh: - test_yml = yaml.safe_load(fh) - - # Verify that tags are correct. All included components in test main.nf should be specified in test.yml so pytests run for all of them - included_components = nf_core.subworkflows.SubworkflowTestYmlBuilder.parse_module_tags( - subworkflow, subworkflow.component_dir - ) - for test in test_yml: - for component in set(included_components): - if component in test["tags"]: - subworkflow.passed.append( - ( - "test_yml_tags", - f"Included module/subworkflow `{component}` specified in `test.yml`", - subworkflow.test_yml, - ) - ) - else: + # Validate no empty files + with open(snap_file, "r") as snap_fh: + snap_content = snap_fh.read() + if "d41d8cd98f00b204e9800998ecf8427e" in snap_content: subworkflow.failed.append( ( - "test_yml_tags", - f"Included module/subworkflow `{component}` missing in `test.yml`", - subworkflow.test_yml, - ) - ) - if component.startswith("subworkflows/"): - included_components += nf_core.subworkflows.SubworkflowTestYmlBuilder.parse_module_tags( - _, - Path(subworkflow.component_dir).parent.joinpath(component.replace("subworkflows/", "")), - ) - included_components = list(set(included_components)) - - # Look for md5sums of empty files - for tfile in test.get("files", []): - if tfile.get("md5sum") == "d41d8cd98f00b204e9800998ecf8427e": - subworkflow.failed.append( - ( - "test_yml_md5sum", + "test_snap_md5sum", "md5sum for empty file found: d41d8cd98f00b204e9800998ecf8427e", - subworkflow.test_yml, + snap_file, ) ) else: subworkflow.passed.append( ( - "test_yml_md5sum", + "test_snap_md5sum", "no md5sum for empty file found", - subworkflow.test_yml, + snap_file, ) ) - if tfile.get("md5sum") == "7029066c27ac6f5ef18d660d5741979a": + if "7029066c27ac6f5ef18d660d5741979a" in snap_content: subworkflow.failed.append( ( - "test_yml_md5sum", + "test_snap_md5sum", "md5sum for compressed empty file found: 7029066c27ac6f5ef18d660d5741979a", - subworkflow.test_yml, + snap_file, ) ) else: subworkflow.passed.append( ( - "test_yml_md5sum", + "test_snap_md5sum", "no md5sum for compressed empty file found", - subworkflow.test_yml, + snap_file, ) ) + else: + subworkflow.failed.append( + ("test_snapshot_exists", "test `main.nf.test.snap` does not exist", snap_file) + ) + # Verify that tags are correct. + main_nf_tags = subworkflow._get_main_nf_tags(fh) + required_tags = [ + "subworkflows", + f"subworkflows/{subworkflow.component_name}", + "subworkflows_nfcore", + subworkflow.component_name, + ] + included_components = [] + if subworkflow.main_nf.is_file(): + included_components = subworkflow._get_included_components(subworkflow.main_nf) + missing_tags = [] + for tag in required_tags + included_components: + if tag not in main_nf_tags: + missing_tags.append(tag) + if len(missing_tags) == 0: + subworkflow.passed.append(("test_main_tags", "Tags adhere to guidelines", subworkflow.nftest_main_nf)) + else: + subworkflow.failed.append( + ( + "test_main_tags", + f"Tags do not adhere to guidelines. Tags missing in `main.nf.test`: {missing_tags}", + subworkflow.nftest_main_nf, + ) + ) - # Test that the file exists - subworkflow.passed.append(("test_yml_exists", "Test `test.yml` exists", subworkflow.test_yml)) + # Check pytest_modules.yml does not contain entries for subworkflows with nf-test + pytest_yml_path = os.path.join(subworkflow.base_dir, "tests", "config", "pytest_modules.yml") + if pytest_yml_path.is_file(): + try: + with open(pytest_yml_path, "r") as fh: + pytest_yml = yaml.safe_load(fh) + if "subworkflows/" + subworkflow.component_name in pytest_yml.keys(): + subworkflow.failed.append( + ( + "test_pytest_yml", + "subworkflow with nf-test should not be listed in pytest_modules.yml", + pytest_yml_path, + ) + ) + else: + subworkflow.passed.append( + ("test_pytest_yml", "subworkflow with nf-test not in pytest_modules.yml", pytest_yml_path) + ) except FileNotFoundError: - subworkflow.failed.append(("test_yml_exists", "Test `test.yml` does not exist", subworkflow.test_yml)) - subworkflow.failed.append(("test_yml_exists", "Test `test.yml` does not exist", subworkflow.test_yml)) - subworkflow.failed.append(("test_yml_exists", "Test `test.yml` does not exist", subworkflow.test_yml)) + subworkflow.warned.append(("test_pytest_yml", "Could not open pytest_modules.yml file", pytest_yml_path)) + + if subworkflow.tags_yml.is_file(): + # Check tags.yml exists and it has the correct entry + subworkflow.passed.append(("test_tags_yml_exists", "file `tags.yml` exists", subworkflow.tags_yml)) + with open(subworkflow.tags_yml, "r") as fh: + tags_yml = yaml.safe_load(fh) + if "subworkflows/" + subworkflow.component_name in tags_yml.keys(): + subworkflow.passed.append(("test_tags_yml", "correct entry in tags.yml", subworkflow.tags_yml)) + if ( + f"subworkflows/{subworkflow.org}/{subworkflow.component_name}/**" + in tags_yml["subworkflows/" + subworkflow.component_name] + ): + subworkflow.passed.append(("test_tags_yml", "correct path in tags.yml", subworkflow.tags_yml)) + else: + subworkflow.failed.append(("test_tags_yml", "incorrect path in tags.yml", subworkflow.tags_yml)) + else: + subworkflow.failed.append( + ( + "test_tags_yml", + "incorrect entry in tags.yml, should be 'subworkflows/'", + subworkflow.tags_yml, + ) + ) + else: + subworkflow.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) From bb070c639d8d960f15e8d3e539889d02c2af3281 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Wed, 15 Nov 2023 18:22:08 +0100 Subject: [PATCH 080/158] fix linting components errors --- nf_core/components/nfcore_component.py | 18 ++++++++++-------- nf_core/module-template/modules/tests/tags.yml | 2 +- nf_core/modules/lint/module_tests.py | 4 ++-- nf_core/modules/lint/module_todos.py | 7 ------- .../subworkflows/tests/main.nf.test | 6 ++++-- nf_core/subworkflows/lint/subworkflow_tests.py | 4 ++-- nf_core/subworkflows/lint/subworkflow_todos.py | 7 ------- 7 files changed, 19 insertions(+), 29 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index e59762e227..94e7584a9b 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -73,18 +73,20 @@ def __init__( self.test_yml = None self.test_main_nf = None - def _get_main_nf_tags(self, test_main_nf): + def _get_main_nf_tags(self, test_main_nf: str): """Collect all tags from the main.nf.test file.""" tags = [] - for line in test_main_nf: - if line.strip().startswith("tag"): - tags.append(line.strip().split()[1].strip('"')) + with open(test_main_nf, "r") as fh: + for line in fh: + if line.strip().startswith("tag"): + tags.append(line.strip().split()[1].strip('"')) return tags - def _get_included_components(self, main_nf): + def _get_included_components(self, main_nf: str): """Collect all included components from the main.nf file.""" included_components = [] - for line in main_nf: - if line.strip().startswith("include"): - included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) + with open(main_nf, "r") as fh: + for line in fh: + if line.strip().startswith("include"): + included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) return included_components diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/modules/tests/tags.yml index 5d60d3a953..e7fac9f5b9 100644 --- a/nf_core/module-template/modules/tests/tags.yml +++ b/nf_core/module-template/modules/tests/tags.yml @@ -1,2 +1,2 @@ -{{ component_name_underscore }}: +{{ component_dir }}: - "modules/{{ org }}/{{ component_dir }}/**" diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index e03f6abc53..54126349ba 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -78,7 +78,7 @@ def module_tests(_, module): ("test_snapshot_exists", "snapshot file `main.nf.test.snap` does not exist", snap_file) ) # Verify that tags are correct. - main_nf_tags = module._get_main_nf_tags(fh) + main_nf_tags = module._get_main_nf_tags(module.nftest_main_nf) required_tags = ["modules", "modules_nfcore", module.component_name] if module.component_name.count("/") == 1: required_tags.append(module.component_name.split("/")[0]) @@ -98,7 +98,7 @@ def module_tests(_, module): ) # Check pytest_modules.yml does not contain entries for subworkflows with nf-test - pytest_yml_path = os.path.join(module.base_dir, "tests", "config", "pytest_modules.yml") + pytest_yml_path = module.base_dir / "tests" / "config" / "pytest_modules.yml" if pytest_yml_path.is_file(): try: with open(pytest_yml_path, "r") as fh: diff --git a/nf_core/modules/lint/module_todos.py b/nf_core/modules/lint/module_todos.py index ee12307512..c9c90ec3dc 100644 --- a/nf_core/modules/lint/module_todos.py +++ b/nf_core/modules/lint/module_todos.py @@ -38,10 +38,3 @@ def module_todos(_, module): module.warned.append(("module_todo", warning, mod_results["file_paths"][i])) for i, passed in enumerate(mod_results["passed"]): module.passed.append(("module_todo", passed, module.component_dir)) - - # Module tests directory - test_results = pipeline_todos(None, root_dir=module.test_dir) - for i, warning in enumerate(test_results["warned"]): - module.warned.append(("module_todo", warning, test_results["file_paths"][i])) - for i, passed in enumerate(test_results["passed"]): - module.passed.append(("module_todo", passed, module.test_dir)) diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index 32f26be546..a0a18e0739 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -10,8 +10,10 @@ nextflow_workflow { tag "subworkflows_nfcore" tag "{{ component_name }}" tag "subworkflows/{{ component_name }}" - - // TODO nf-core: Add tags for all modules used within this subworkflow + // TODO nf-core: Add tags for all modules used within this subworkflow. Example: + tag "samtools" + tag "samtools/sort" + tag "samtools/index" // TODO nf-core: Change the test name preferably indicating the test-data and file-format used diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8a9429e151..f457841c99 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -82,7 +82,7 @@ def subworkflow_tests(_, subworkflow): ("test_snapshot_exists", "test `main.nf.test.snap` does not exist", snap_file) ) # Verify that tags are correct. - main_nf_tags = subworkflow._get_main_nf_tags(fh) + main_nf_tags = subworkflow._get_main_nf_tags(subworkflow.nftest_main_nf) required_tags = [ "subworkflows", f"subworkflows/{subworkflow.component_name}", @@ -108,7 +108,7 @@ def subworkflow_tests(_, subworkflow): ) # Check pytest_modules.yml does not contain entries for subworkflows with nf-test - pytest_yml_path = os.path.join(subworkflow.base_dir, "tests", "config", "pytest_modules.yml") + pytest_yml_path = subworkflow.base_dir / "tests" / "config" / "pytest_modules.yml" if pytest_yml_path.is_file(): try: with open(pytest_yml_path, "r") as fh: diff --git a/nf_core/subworkflows/lint/subworkflow_todos.py b/nf_core/subworkflows/lint/subworkflow_todos.py index 1de02b18e2..91f9f55b0b 100644 --- a/nf_core/subworkflows/lint/subworkflow_todos.py +++ b/nf_core/subworkflows/lint/subworkflow_todos.py @@ -38,10 +38,3 @@ def subworkflow_todos(_, subworkflow): subworkflow.warned.append(("subworkflow_todo", warning, swf_results["file_paths"][i])) for i, passed in enumerate(swf_results["passed"]): subworkflow.passed.append(("subworkflow_todo", passed, subworkflow.component_dir)) - - # Module tests directory - test_results = pipeline_todos(None, root_dir=subworkflow.test_dir) - for i, warning in enumerate(test_results["warned"]): - subworkflow.warned.append(("subworkflow_todo", warning, test_results["file_paths"][i])) - for i, passed in enumerate(test_results["passed"]): - subworkflow.passed.append(("subworkflow_todo", passed, subworkflow.test_dir)) From b4057ca4c50aa3b864076c0fc0fe8c1d687e9592 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 16 Nov 2023 09:14:19 +0100 Subject: [PATCH 081/158] Update slackreport.json No need for the extra `v` cf https://github.com/nf-core/sarek/pull/1334 and the rnaseq pipeline --- nf_core/pipeline-template/assets/slackreport.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/assets/slackreport.json b/nf_core/pipeline-template/assets/slackreport.json index ec03b3968a..96d2cb8afc 100644 --- a/nf_core/pipeline-template/assets/slackreport.json +++ b/nf_core/pipeline-template/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "{{ name }} v${version} - ${runName}", + "author_name": "{{ name }} ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ From fa5e84fd6101ccc383acf4e86b095be3b8915edd Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 11:06:18 +0100 Subject: [PATCH 082/158] add licence to mock anaconda call --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index d39d172a66..dfc743f9cc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -83,7 +83,7 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): "doc_url": "http://test", "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], - "license": "", + "license": "MIT", } rsps.get(anaconda_api_url, json=anaconda_mock, status=200) From 3b03e4c0f79e9726a198cf6e4f2d68e89a527d82 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 13:01:49 +0100 Subject: [PATCH 083/158] fix licence type again --- tests/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils.py b/tests/utils.py index dfc743f9cc..1357c71428 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -83,7 +83,7 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): "doc_url": "http://test", "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], - "license": "MIT", + "license": ["MIT"], } rsps.get(anaconda_api_url, json=anaconda_mock, status=200) From d9d8eb05206e4070aae03c9ccd848dcdf83264ba Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 13:45:53 +0100 Subject: [PATCH 084/158] give better hint, fix tests --- nf_core/module-template/modules/meta.yml | 2 +- nf_core/modules/lint/meta_yml.py | 5 +++++ tests/utils.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index aea3c36aa3..495760ba02 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -19,7 +19,7 @@ tools: documentation: "{{ tool_doc_url }}" tool_dev_url: "{{ tool_dev_url }}" doi: "" - licence: "{{ tool_licence }}" + licence: {{ tool_licence }} {% if not_empty_template -%} ## TODO nf-core: Add a description of all of the variables used as input diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 69fb90c3f9..218db0004f 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -58,6 +58,11 @@ def meta_yml(module_lint_object, module): hint = f"\nCheck that the child entries of {e.path[0]+'.'+e.path[2]} are indented correctly." if e.schema.get("message"): e.message = e.schema["message"] + incorrect_value = meta_yaml + for key in e.path: + incorrect_value = incorrect_value[key] + + hint = hint + f"\nThe current value is `{incorrect_value}`." module.failed.append( ( "meta_yml_valid", diff --git a/tests/utils.py b/tests/utils.py index 1357c71428..dfc743f9cc 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -83,7 +83,7 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): "doc_url": "http://test", "dev_url": "http://test", "files": [{"version": version.split("--")[0]}], - "license": ["MIT"], + "license": "MIT", } rsps.get(anaconda_api_url, json=anaconda_mock, status=200) From e19a4a250bf7a8a4ebd484f94cc39b324642081f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 16 Nov 2023 12:58:10 +0000 Subject: [PATCH 085/158] remove tmp dir manually if we find a PermissionError --- tests/test_components.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_components.py b/tests/test_components.py index d2b0c4630f..ed84f85b4d 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -3,6 +3,7 @@ import os import shutil +import subprocess import tempfile import unittest from pathlib import Path @@ -33,7 +34,10 @@ def tearDown(self): # Clean up temporary files if self.tmp_dir.is_dir(): - shutil.rmtree(self.tmp_dir) + try: + shutil.rmtree(self.tmp_dir) + except PermissionError: + subprocess.run(["rm", "-rf", self.tmp_dir]) ############################################ # Test of the individual components commands. # From 5e0261941279843a2613903e9e0fc2c7ec259df2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 14:38:48 +0100 Subject: [PATCH 086/158] fix launch tests, move tmp pipeline creation to utils to share between tests add types to utils --- tests/test_launch.py | 35 +++++++++++++++-------------------- tests/test_modules.py | 10 ++-------- tests/test_subworkflows.py | 14 ++------------ tests/utils.py | 33 ++++++++++++++++++++++++++------- 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/tests/test_launch.py b/tests/test_launch.py index d830311ba3..03c6a8b692 100644 --- a/tests/test_launch.py +++ b/tests/test_launch.py @@ -3,40 +3,35 @@ import json import os -import tempfile -import unittest -from unittest import mock +import shutil +from pathlib import Path +from unittest import TestCase, mock import pytest import nf_core.create import nf_core.launch -from .utils import with_temporary_file, with_temporary_folder +from .utils import create_tmp_pipeline, with_temporary_file, with_temporary_folder -class TestLaunch(unittest.TestCase): +class TestLaunch(TestCase): """Class for launch tests""" def setUp(self): """Create a new PipelineSchema and Launch objects""" - # Set up the schema - root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - # cannot use a context manager here, since outside setUp the temporary - # file will never exists - self.tmp_dir = tempfile.mkdtemp() + self.tmp_dir, self.template_dir, self.pipeline_name, self.pipeline_dir = create_tmp_pipeline() self.nf_params_fn = os.path.join(self.tmp_dir, "nf-params.json") - self.launcher = nf_core.launch.Launch(self.template_dir, params_out=self.nf_params_fn) + self.launcher = nf_core.launch.Launch(self.pipeline_dir, params_out=self.nf_params_fn) def tearDown(self): """Clean up temporary files and folders""" - if os.path.exists(self.nf_params_fn): - os.remove(self.nf_params_fn) + if Path(self.nf_params_fn).exists(): + Path(self.nf_params_fn).unlink() - if os.path.exists(self.tmp_dir): - os.rmdir(self.tmp_dir) + if Path(self.tmp_dir).exists(): + shutil.rmtree(self.tmp_dir) @mock.patch.object(nf_core.launch.Launch, "prompt_web_gui", side_effect=[True]) @mock.patch.object(nf_core.launch.Launch, "launch_web_gui") @@ -304,7 +299,7 @@ def test_build_command_empty(self): self.launcher.get_pipeline_schema() self.launcher.merge_nxf_flag_schema() self.launcher.build_command() - assert self.launcher.nextflow_cmd == f"nextflow run {self.template_dir}" + assert self.launcher.nextflow_cmd == f"nextflow run {self.pipeline_dir}" def test_build_command_nf(self): """Test the functionality to build a nextflow command - core nf customised""" @@ -313,7 +308,7 @@ def test_build_command_nf(self): self.launcher.nxf_flags["-name"] = "Test_Workflow" self.launcher.nxf_flags["-resume"] = True self.launcher.build_command() - assert self.launcher.nextflow_cmd == f'nextflow run {self.template_dir} -name "Test_Workflow" -resume' + assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} -name "Test_Workflow" -resume' def test_build_command_params(self): """Test the functionality to build a nextflow command - params supplied""" @@ -323,7 +318,7 @@ def test_build_command_params(self): # Check command assert ( self.launcher.nextflow_cmd - == f'nextflow run {self.template_dir} -params-file "{os.path.relpath(self.nf_params_fn)}"' + == f'nextflow run {self.pipeline_dir} -params-file "{os.path.relpath(self.nf_params_fn)}"' ) # Check saved parameters file with open(self.nf_params_fn, "r") as fh: @@ -340,4 +335,4 @@ def test_build_command_params_cl(self): self.launcher.get_pipeline_schema() self.launcher.schema_obj.input_params.update({"input": "custom_input"}) self.launcher.build_command() - assert self.launcher.nextflow_cmd == f'nextflow run {self.template_dir} --input "custom_input"' + assert self.launcher.nextflow_cmd == f'nextflow run {self.pipeline_dir} --input "custom_input"' diff --git a/tests/test_modules.py b/tests/test_modules.py index 01c858c136..a6ae779f18 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -20,6 +20,7 @@ GITLAB_URL, OLD_TRIMGALORE_BRANCH, OLD_TRIMGALORE_SHA, + create_tmp_pipeline, mock_anaconda_api_calls, mock_biocontainers_api_calls, ) @@ -63,17 +64,10 @@ class TestModules(unittest.TestCase): def setUp(self): """Create a new PipelineSchema and Launch objects""" - self.tmp_dir = tempfile.mkdtemp() self.component_type = "modules" # Set up the schema - root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = Path(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_name = "mypipeline" - self.pipeline_dir = Path(self.tmp_dir, self.pipeline_name) - nf_core.create.PipelineCreate( - self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True - ).init_pipeline() + self.tmp_dir, self.template_dir, self.pipeline_name, self.pipeline_dir = create_tmp_pipeline() # Set up install objects self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_old = nf_core.modules.ModuleInstall( diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index b499848409..925e70fb76 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -3,11 +3,8 @@ import os import shutil -import tempfile import unittest -import responses - import nf_core.create import nf_core.modules import nf_core.subworkflows @@ -17,6 +14,7 @@ GITLAB_SUBWORKFLOWS_ORG_PATH_BRANCH, GITLAB_URL, OLD_SUBWORKFLOWS_SHA, + create_tmp_pipeline, ) @@ -47,18 +45,10 @@ class TestSubworkflows(unittest.TestCase): def setUp(self): """Create a new PipelineStructure and Launch objects""" - self.tmp_dir = tempfile.mkdtemp() self.component_type = "subworkflows" # Set up the pipeline structure - root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - self.template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") - self.pipeline_name = "mypipeline" - self.pipeline_dir = os.path.join(self.tmp_dir, self.pipeline_name) - nf_core.create.PipelineCreate( - self.pipeline_name, "it is mine", "me", no_git=True, outdir=self.pipeline_dir, plain=True - ).init_pipeline() - + self.tmp_dir, self.template_dir, self.pipeline_name, self.pipeline_dir = create_tmp_pipeline() # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir) diff --git a/tests/utils.py b/tests/utils.py index 5c26485bb1..307129b5b2 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,9 +7,11 @@ import tempfile from contextlib import contextmanager from pathlib import Path +from typing import Any, Callable, Generator, Tuple import responses +import nf_core.create import nf_core.modules OLD_TRIMGALORE_SHA = "9b7a3bdefeaad5d42324aa7dd50f87bea1b04386" @@ -28,7 +30,7 @@ GITLAB_NFTEST_BRANCH = "nf-test-tests" -def with_temporary_folder(func): +def with_temporary_folder(func: Callable[..., Any]) -> Callable[..., Any]: """ Call the decorated function under the tempfile.TemporaryDirectory context manager. Pass the temporary directory name to the decorated @@ -36,21 +38,21 @@ def with_temporary_folder(func): """ @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: with tempfile.TemporaryDirectory() as tmpdirname: return func(*args, tmpdirname, **kwargs) return wrapper -def with_temporary_file(func): +def with_temporary_file(func: Callable[..., Any]) -> Callable[..., Any]: """ Call the decorated function under the tempfile.NamedTemporaryFile context manager. Pass the opened file handle to the decorated function """ @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Any: with tempfile.NamedTemporaryFile() as tmpfile: return func(*args, tmpfile, **kwargs) @@ -58,7 +60,7 @@ def wrapper(*args, **kwargs): @contextmanager -def set_wd(path: Path): +def set_wd(path: Path) -> Generator[None, None, None]: """Sets the working directory for this context. Arguments @@ -75,7 +77,7 @@ def set_wd(path: Path): os.chdir(start_wd) -def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): +def mock_anaconda_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" anaconda_mock = { @@ -89,7 +91,7 @@ def mock_anaconda_api_calls(rsps: responses.RequestsMock, module, version): rsps.get(anaconda_api_url, json=anaconda_mock, status=200) -def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): +def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock biocontainers api calls for module""" biocontainers_api_url = ( f"https://api.biocontainers.pro/ga4gh/trs/v2/tools/{module}/versions/{module}-{version.split('--')[0]}" @@ -109,3 +111,20 @@ def mock_biocontainers_api_calls(rsps: responses.RequestsMock, module, version): ], } rsps.get(biocontainers_api_url, json=biocontainers_mock, status=200) + + +def create_tmp_pipeline() -> Tuple[str, str, str, str]: + """Create a new Pipeline for testing""" + + tmp_dir = tempfile.mkdtemp() + root_repo_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + template_dir = os.path.join(root_repo_dir, "nf_core", "pipeline-template") + pipeline_name = "mypipeline" + pipeline_dir = os.path.join(tmp_dir, pipeline_name) + + nf_core.create.PipelineCreate( + pipeline_name, "it is mine", "me", no_git=True, outdir=pipeline_dir, plain=True + ).init_pipeline() + + # return values to instance variables for later use in test methods + return tmp_dir, template_dir, pipeline_name, pipeline_dir From 7b0683ea6c16a940117d19c48888040a2e0d7410 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 15:03:43 +0100 Subject: [PATCH 087/158] fix tests by correcting raised error type --- tests/test_list.py | 2 +- tests/test_schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_list.py b/tests/test_list.py index 70af3fada5..c1f51e03e0 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -65,7 +65,7 @@ def test_local_workflows_and_fail(self): """Test the local workflow class and try to get local Nextflow workflow information""" loc_wf = nf_core.list.LocalWorkflow("myWF") - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError): loc_wf.get_local_nf_workflow_details() def test_local_workflows_compare_and_fail_silently(self): diff --git a/tests/test_schema.py b/tests/test_schema.py index d3b4fda817..105cd9473e 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -47,7 +47,7 @@ def test_load_lint_schema(self): def test_load_lint_schema_nofile(self): """Check that linting raises properly if a non-existant file is given""" - with pytest.raises(AssertionError): + with pytest.raises(RuntimeError): self.schema_obj.get_schema_path("fake_file") def test_load_lint_schema_notjson(self): From 22a907118d4cbb3db48e373a2796d193531613c6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 16 Nov 2023 16:19:59 +0100 Subject: [PATCH 088/158] fix tests, switch to pdiff instead of icdiff, because it handles width better --- nf_core/components/snapshot_generator.py | 4 ++-- requirements.txt | 2 +- tests/components/create_snapshot.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py index 9b73387963..51013d570e 100644 --- a/nf_core/components/snapshot_generator.py +++ b/nf_core/components/snapshot_generator.py @@ -55,10 +55,10 @@ def __init__( def run(self) -> None: """Run build steps""" self.check_inputs() - os.environ["NFT_DIFF"] = "icdiff" # set nf-test differ to icdiff to get a better diff output + os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output os.environ[ "NFT_DIFF_ARGS" - ] = "-N --cols 120 -L old_snapshot -L new_snapshot" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences with set_wd(self.dir): self.check_snapshot_stability() if len(self.errors) > 0: diff --git a/requirements.txt b/requirements.txt index ace9b44a5e..add52f4bc6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,4 +17,4 @@ requests_cache rich-click>=1.6.1 rich>=13.3.1 tabulate -icdiff +pdiff diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index e8632536f1..a18738da06 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -100,7 +100,7 @@ def test_test_not_found(self): ) with pytest.raises(UserWarning) as e: snap_generator.run() - assert "Test file 'main.nf.test' not found" in str(e.value) + assert "Test file 'main.nf.test' not found" in str(e.value) def test_unstable_snapshot(self): @@ -115,4 +115,4 @@ def test_unstable_snapshot(self): ) with pytest.raises(UserWarning) as e: snap_generator.run() - assert "nf-test snapshot is not stable" in str(e.value) + assert "nf-test snapshot is not stable" in str(e.value) From a8f6332c4b0648f33d66ac2e79941182279c4028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 16 Nov 2023 15:58:28 +0000 Subject: [PATCH 089/158] revert e19a4a250bf7a8a4ebd484f94cc39b324642081f --- tests/test_components.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_components.py b/tests/test_components.py index ed84f85b4d..d2b0c4630f 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -3,7 +3,6 @@ import os import shutil -import subprocess import tempfile import unittest from pathlib import Path @@ -34,10 +33,7 @@ def tearDown(self): # Clean up temporary files if self.tmp_dir.is_dir(): - try: - shutil.rmtree(self.tmp_dir) - except PermissionError: - subprocess.run(["rm", "-rf", self.tmp_dir]) + shutil.rmtree(self.tmp_dir) ############################################ # Test of the individual components commands. # From 5347b565b245717de8ed1ade56e39121b27a4749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 16 Nov 2023 17:42:34 +0100 Subject: [PATCH 090/158] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/__main__.py | 6 ++---- nf_core/modules/lint/module_tests.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0e2d6edb4e..d2f7d75208 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1090,10 +1090,8 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update): """ - Auto-generate a test.yml file for a new subworkflow. - - Given the name of a module, runs the Nextflow test command and automatically generate - the required `test.yml` file based on the output files. + Generate nf-test snapshots for a module. + Given the name of a module, runs the nf-test command to generate snapshots. """ from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 54126349ba..3dffefeaa9 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -97,7 +97,7 @@ def module_tests(_, module): ) ) - # Check pytest_modules.yml does not contain entries for subworkflows with nf-test + # Check pytest_modules.yml does not contain entries for modules with nf-test pytest_yml_path = module.base_dir / "tests" / "config" / "pytest_modules.yml" if pytest_yml_path.is_file(): try: From fc0a3fcb4c2a97a11c6eeb63b01df68f61b048f3 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Thu, 16 Nov 2023 17:45:20 +0100 Subject: [PATCH 091/158] remove debugging raise introduced by mistake --- nf_core/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d2f7d75208..8f117ce741 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1351,7 +1351,6 @@ def install(ctx, subworkflow, dir, prompt, force, sha): sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) - raise sys.exit(1) From c576cc87cf29c700ad3034422b6bc317d568c395 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 17 Nov 2023 15:01:27 +0100 Subject: [PATCH 092/158] fix module name in the version of stub scripts --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 5ffef06bb6..a30221b4ee 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -113,7 +113,7 @@ process {{ component_name_underscore|upper }} { cat <<-END_VERSIONS > versions.yml "${task.process}": - {{ tool }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) + {{ component }}: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' )) END_VERSIONS """ } From 2ac549db200e171796f0e22130b631cf31037274 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 17 Nov 2023 15:01:41 +0100 Subject: [PATCH 093/158] add maintainers to meta.yml --- nf_core/module-template/modules/meta.yml | 2 ++ nf_core/subworkflow-template/subworkflows/meta.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index aea3c36aa3..857f0f29f2 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -65,3 +65,5 @@ output: authors: - "{{ author }}" +maintainers: + - "{{ author }}" diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/subworkflows/meta.yml index cb77dc23a3..7c83b3c490 100644 --- a/nf_core/subworkflow-template/subworkflows/meta.yml +++ b/nf_core/subworkflow-template/subworkflows/meta.yml @@ -47,3 +47,5 @@ output: pattern: "versions.yml" authors: - "{{ author }}" +maintainers: + - "{{ author }}" From d12560d25535eae169790b8dca85b3b1c42cfd85 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Nov 2023 11:40:23 +0100 Subject: [PATCH 094/158] move create-snapshot command to test command --- nf_core/__main__.py | 60 +--- nf_core/components/components_test.py | 319 +++++++++++++--------- nf_core/components/snapshot_generator.py | 217 --------------- nf_core/modules/__init__.py | 1 - nf_core/modules/modules_test.py | 30 -- nf_core/subworkflows/__init__.py | 1 - nf_core/subworkflows/subworkflows_test.py | 31 --- tests/components/create_snapshot.py | 14 +- tests/components/snapshot_test.py | 40 +++ tests/subworkflows/subworkflows_test.py | 18 -- tests/test_modules.py | 5 - tests/test_subworkflows.py | 5 - 12 files changed, 240 insertions(+), 501 deletions(-) delete mode 100644 nf_core/components/snapshot_generator.py delete mode 100644 nf_core/modules/modules_test.py delete mode 100644 nf_core/subworkflows/subworkflows_test.py create mode 100644 tests/components/snapshot_test.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 8f117ce741..7e82b08990 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -851,8 +851,8 @@ def create_module( sys.exit(1) -# nf-core modules create-snapshot -@modules.command("create-snapshot") +# nf-core modules test +@modules.command("test") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -865,10 +865,10 @@ def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update): Given the name of a module, runs the nf-test command to generate snapshots. """ - from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator + from nf_core.components.components_test import ComponentsTest try: - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="modules", component_name=tool, directory=dir, @@ -1027,28 +1027,6 @@ def bump_versions(ctx, tool, dir, all, show_all): sys.exit(1) -# nf-core modules test -@modules.command("test") -@click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") -@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments") -def test_module(ctx, tool, no_prompts, pytest_args): - """ - Run module tests locally. - - Given the name of a module, runs the Nextflow test command. - """ - from nf_core.modules import ModulesTest - - try: - meta_builder = ModulesTest(tool, no_prompts, pytest_args) - meta_builder.run() - except (UserWarning, LookupError) as e: - log.critical(e) - sys.exit(1) - - # nf-core subworkflows create @subworkflows.command("create") @click.pass_context @@ -1080,8 +1058,8 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): sys.exit(1) -# nf-core subworkflows create-snapshot -@subworkflows.command("create-snapshot") +# nf-core subworkflows test +@subworkflows.command("test") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -1093,10 +1071,10 @@ def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update): Generate nf-test snapshots for a module. Given the name of a module, runs the nf-test command to generate snapshots. """ - from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator + from nf_core.components.components_test import ComponentsTest try: - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="subworkflows", component_name=subworkflow, directory=dir, @@ -1290,28 +1268,6 @@ def info(ctx, tool, dir): sys.exit(1) -# nf-core subworkflows test -@subworkflows.command("test") -@click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") -@click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") -@click.option("-a", "--pytest_args", type=str, required=False, multiple=True, help="Additional pytest arguments") -def test_subworkflow(ctx, subworkflow, no_prompts, pytest_args): - """ - Run subworkflow tests locally. - - Given the name of a subworkflow, runs the Nextflow test command. - """ - from nf_core.subworkflows import SubworkflowsTest - - try: - meta_builder = SubworkflowsTest(subworkflow, no_prompts, pytest_args) - meta_builder.run() - except (UserWarning, LookupError) as e: - log.critical(e) - sys.exit(1) - - # nf-core subworkflows install @subworkflows.command() @click.pass_context diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 37e275aea8..3470fbe4a1 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -1,200 +1,251 @@ +""" +The ComponentsTest class handles the generation and testing of nf-test snapshots. +""" + +from __future__ import print_function + import logging import os -import sys +import re from pathlib import Path -from shutil import which +from typing import List, Optional -import pytest import questionary -import rich -from git import InvalidGitRepositoryError, Repo +from rich import print +from rich.panel import Panel +from rich.prompt import Confirm +from rich.syntax import Syntax +from rich.text import Text -import nf_core.modules.modules_utils import nf_core.utils from nf_core.components.components_command import ComponentCommand -from nf_core.modules.modules_json import ModulesJson +from tests.utils import set_wd log = logging.getLogger(__name__) class ComponentsTest(ComponentCommand): """ - Class to run module and subworkflow pytests. + Class to generate and test nf-test snapshots for modules. ... Attributes ---------- + component_type : str + type of component to test (modules or subworkflows) component_name : str name of the tool to run tests for + directory: str + path to modules repository directory + run_tests : bool + flag indicating if tests should be run no_prompts : bool flat indicating if prompts are used - pytest_args : tuple - additional arguments passed to pytest command + remote_url : str + URL of the remote repository + branch : str + branch of the remote repository + verbose : bool + flag indicating if verbose output should be used + update : bool + flag indicating if the existing snapshot should be updated Methods ------- run(): Run test steps - _check_inputs(): + check_inputs(): Check inputs. Ask for component_name if not provided and check that the directory exists - _set_profile(): - Set software profile - _run_pytests(self): - Run pytest + generate_snapshot(): + Generate the nf-test snapshot using `nf-test test` command + check_snapshot_stability(): + Run the nf-test twice and check if the snapshot changes """ def __init__( self, - component_type, - component_name=None, - no_prompts=False, - pytest_args="", - remote_url=None, - branch=None, - no_pull=False, + component_type: str, + component_name: Optional[str] = None, + directory: str = ".", + run_tests: bool = False, + no_prompts: bool = False, + remote_url: Optional[str] = None, + branch: Optional[str] = None, + verbose: bool = False, + update: bool = False, ): - super().__init__(component_type=component_type, dir=".", remote_url=remote_url, branch=branch, no_pull=no_pull) + super().__init__(component_type, directory, remote_url, branch) self.component_name = component_name + self.remote_url = remote_url + self.branch = branch + self.run_tests = run_tests self.no_prompts = no_prompts - self.pytest_args = pytest_args - - def run(self): - """Run test steps""" - if not self.no_prompts: - log.info( - "[yellow]Press enter to use default values [cyan bold](shown in brackets) [yellow]or type your own responses" - ) - self._check_inputs() - self._set_profile() - self._check_profile() - self._run_pytests() + self.errors: List[str] = [] + self.verbose = verbose + self.obsolete_snapshots: bool = False + self.update = update + + def run(self) -> None: + """Run build steps""" + self.check_inputs() + os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output + os.environ[ + "NFT_DIFF_ARGS" + ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences + with set_wd(self.dir): + self.check_snapshot_stability() + if len(self.errors) > 0: + errors = "\n - ".join(self.errors) + raise UserWarning(f"Ran, but found errors:\n - {errors}") + else: + log.info("All tests passed!") - def _check_inputs(self): + def check_inputs(self) -> None: """Do more complex checks about supplied flags.""" # Check modules directory structure - self.check_modules_structure() - - # Retrieving installed modules - if self.repo_type == "modules": - installed_components = self.get_components_clone_modules() - else: - modules_json = ModulesJson(self.dir) - modules_json.check_up_to_date() - installed_components = modules_json.get_all_components(self.component_type).get( - self.modules_repo.remote_url - ) + if self.component_type == "modules": + self.check_modules_structure() # Get the component name if not specified if self.component_name is None: - if self.no_prompts: - raise UserWarning( - f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}." - ) - if not installed_components: - if self.component_type == "modules": - dir_structure_message = f"modules/{self.modules_repo.repo_path}/TOOL/SUBTOOL/ and tests/modules/{self.modules_repo.repo_path}/TOOLS/SUBTOOL/" - elif self.component_type == "subworkflows": - dir_structure_message = f"subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/ and tests/subworkflows/{self.modules_repo.repo_path}/SUBWORKFLOW/" - raise UserWarning( - f"No installed {self.component_type} were found from '{self.modules_repo.remote_url}'.\n" - f"Are you running the tests inside the repository root directory?\n" - f"Make sure that the directory structure is {dir_structure_message}" - ) self.component_name = questionary.autocomplete( - f"{self.component_type[:-1]} name:", - choices=installed_components, + "Tool name:" if self.component_type == "modules" else "Subworkflow name:", + choices=self.components_from_repo(self.org), style=nf_core.utils.nfcore_question_style, ).unsafe_ask() - # Sanity check that the module directory exists - self._validate_folder_structure() + self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) - def _validate_folder_structure(self): - """Validate that the modules follow the correct folder structure to run the tests: - - modules/nf-core/TOOL/SUBTOOL/ - - tests/modules/nf-core/TOOL/SUBTOOL/ - or - - subworkflows/nf-core/SUBWORKFLOW/ - - tests/subworkflows/nf-core/SUBWORKFLOW/ - """ - if self.component_type == "modules": - component_path = Path(self.default_modules_path) / self.component_name - test_path = Path(self.default_tests_path) / self.component_name - elif self.component_type == "subworkflows": - component_path = Path(self.default_subworkflows_path) / self.component_name - test_path = Path(self.default_subworkflows_tests_path) / self.component_name - - if not (self.dir / component_path).is_dir(): + # First, sanity check that the module directory exists + if not Path(self.dir, self.component_dir).is_dir(): raise UserWarning( - f"Cannot find directory '{component_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. Are you running the tests inside the modules repository root directory?" - ) - if not (self.dir / test_path).is_dir(): - raise UserWarning( - f"Cannot find directory '{test_path}'. Should be {'TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else 'SUBWORKFLOW'}. " - "Are you running the tests inside the modules repository root directory? " - "Do you have tests for the specified module?" + f"Cannot find directory '{self.component_dir}'.{' Should be TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}" ) - def _set_profile(self): - """Set $PROFILE env variable. - The config expects $PROFILE and Nextflow fails if it's not set. - """ + # Check that we're running tests if no prompts + if not self.run_tests and self.no_prompts: + log.debug("Setting run_tests to True as running without prompts") + self.run_tests = True + + # Check container software to use if os.environ.get("PROFILE") is None: os.environ["PROFILE"] = "" if self.no_prompts: log.info( - "Setting environment variable '$PROFILE' to an empty string as not set.\n" - "Tests will run with Docker by default. " + "Setting env var '$PROFILE' to Docker as not set.\n" "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." ) + os.environ["PROFILE"] = "docker" else: question = { "type": "list", "name": "profile", - "message": "Choose software profile", + "message": "Choose container software to run the test with", "choices": ["Docker", "Singularity", "Conda"], } answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) profile = answer["profile"].lower() os.environ["PROFILE"] = profile - log.info(f"Setting environment variable '$PROFILE' to '{profile}'") - - def _check_profile(self): - """Check if profile is available""" - profile = os.environ.get("PROFILE") - # Make sure the profile read from the environment is a valid Nextflow profile. - valid_nextflow_profiles = ["docker", "singularity", "conda"] - if profile in valid_nextflow_profiles: - if not which(profile): - raise UserWarning(f"Command '{profile}' not found - is it installed?") - else: - raise UserWarning( - f"The PROFILE '{profile}' set in the shell environment is not valid.\n" - f"Valid Nextflow profiles are '{', '.join(valid_nextflow_profiles)}'." - ) - def _run_pytests(self): - """Given a module/subworkflow name, run tests.""" - # Print nice divider line - console = rich.console.Console() - console.rule(self.component_name, style="black") - - # Check uncommitted changed - try: - repo = Repo(self.dir) - if repo.is_dirty(): - log.warning("You have uncommitted changes. Make sure to commit last changes before running the tests.") - except InvalidGitRepositoryError: - pass - - # Set pytest arguments - tag = self.component_name - if self.component_type == "subworkflows": - tag = "subworkflows/" + tag - command_args = ["--tag", f"{tag}", "--symlink", "--keep-workflow-wd", "--git-aware"] - command_args += self.pytest_args - - # Run pytest - log.info(f"Running pytest for {self.component_type[:-1]} '{self.component_name}'") - sys.exit(pytest.main(command_args)) + def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: + nftest_output = Text.from_ansi(nftest_out.decode()) + print(Panel(nftest_output, title="nf-test output")) + if nftest_err: + syntax = Syntax(nftest_err.decode(), "diff", theme="ansi_dark") + print(Panel(syntax, title="nf-test error")) + if "Different Snapshot:" in nftest_err.decode(): + log.error("nf-test failed due to differences in the snapshots") + # prompt to update snapshot + if self.no_prompts: + log.info("Updating snapshot") + self.update = True + else: + answer = Confirm.ask( + "[bold][blue]?[/] nf-test found differences in the snapshot. Do you want to update it?", + default=True, + ) + if answer: + log.info("Updating snapshot") + self.update = True + else: + log.debug("Snapshot not updated") + if self.update: + # update snapshot using nf-test --update-snapshot + self.generate_snapshot() + + else: + self.errors.append("nf-test failed") + + def generate_snapshot(self) -> bool: + """Generate the nf-test snapshot using `nf-test test` command + + returns True if the test was successful, False otherwise + """ + + log.debug("Running nf-test test") + + # set verbose flag if self.verbose is True + verbose = "--verbose --debug" if self.verbose else "" + update = "--update-snapshot" if self.update else "" + self.update = False # reset self.update to False to test if the new snapshot is stable + + result = nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", + ) + if result is not None: + nftest_out, nftest_err = result + self.display_nftest_output(nftest_out, nftest_err) + # check if nftest_out contains obsolete snapshots + pattern = r"Snapshot Summary:.*?(\d+)\s+obsolete" + compiled_pattern = re.compile(pattern, re.DOTALL) # re.DOTALL to allow . to match newlines + obsolete_snapshots = compiled_pattern.search(nftest_out.decode()) + if obsolete_snapshots: + self.obsolete_snapshots = True + + # check if nf-test was successful + if "Assertion failed:" in nftest_out.decode(): + return False + elif "no valid tests found." in nftest_out.decode(): + log.error("Test file 'main.nf.test' not found") + self.errors.append("Test file 'main.nf.test' not found") + return False + else: + log.debug("nf-test successful") + return True + else: + log.error("nf-test failed") + self.errors.append("nf-test failed") + return False + + def check_snapshot_stability(self) -> bool: + """Run the nf-test twice and check if the snapshot changes""" + log.info("Generating nf-test snapshot") + if not self.generate_snapshot(): + return False # stop here if the first run failed + log.info("Generating nf-test snapshot again to check stability") + if not self.generate_snapshot(): + log.error("nf-test snapshot is not stable") + self.errors.append("nf-test snapshot is not stable") + return False + else: + if self.obsolete_snapshots: + # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot + if self.no_prompts: + log.info("Removing obsolete snapshots") + nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + ) + else: + answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) + if answer: + log.info("Removing obsolete snapshots") + nf_core.utils.run_cmd( + "nf-test", + f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", + ) + else: + log.debug("Obsolete snapshots not removed") + return True diff --git a/nf_core/components/snapshot_generator.py b/nf_core/components/snapshot_generator.py deleted file mode 100644 index 51013d570e..0000000000 --- a/nf_core/components/snapshot_generator.py +++ /dev/null @@ -1,217 +0,0 @@ -""" -The ComponentTestSnapshotGenerator class handles the generation nf-test snapshots. -""" - -from __future__ import print_function - -import logging -import os -import re -from pathlib import Path -from typing import List, Optional - -import questionary -from rich import print -from rich.panel import Panel -from rich.prompt import Confirm -from rich.syntax import Syntax -from rich.text import Text - -import nf_core.utils -from nf_core.components.components_command import ComponentCommand -from tests.utils import set_wd - -log = logging.getLogger(__name__) - - -class ComponentTestSnapshotGenerator(ComponentCommand): - """ - Class to generate nf-test snapshots for modules. - """ - - def __init__( - self, - component_type: str, - component_name: Optional[str] = None, - directory: str = ".", - run_tests: bool = False, - no_prompts: bool = False, - remote_url: Optional[str] = None, - branch: Optional[str] = None, - verbose: bool = False, - update: bool = False, - ): - super().__init__(component_type, directory, remote_url, branch) - self.component_name = component_name - self.remote_url = remote_url - self.branch = branch - self.run_tests = run_tests - self.no_prompts = no_prompts - self.errors: List[str] = [] - self.verbose = verbose - self.obsolete_snapshots: bool = False - self.update = update - - def run(self) -> None: - """Run build steps""" - self.check_inputs() - os.environ["NFT_DIFF"] = "pdiff" # set nf-test differ to pdiff to get a better diff output - os.environ[ - "NFT_DIFF_ARGS" - ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences - with set_wd(self.dir): - self.check_snapshot_stability() - if len(self.errors) > 0: - errors = "\n - ".join(self.errors) - raise UserWarning(f"Ran, but found errors:\n - {errors}") - else: - log.info("All tests passed!") - - def check_inputs(self) -> None: - """Do more complex checks about supplied flags.""" - # Check modules directory structure - if self.component_type == "modules": - self.check_modules_structure() - - # Get the component name if not specified - if self.component_name is None: - self.component_name = questionary.autocomplete( - "Tool name:" if self.component_type == "modules" else "Subworkflow name:", - choices=self.components_from_repo(self.org), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) - - # First, sanity check that the module directory exists - if not Path(self.dir, self.component_dir).is_dir(): - raise UserWarning( - f"Cannot find directory '{self.component_dir}'.{' Should be TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}" - ) - - # Check that we're running tests if no prompts - if not self.run_tests and self.no_prompts: - log.debug("Setting run_tests to True as running without prompts") - self.run_tests = True - - # Check container software to use - if os.environ.get("PROFILE") is None: - os.environ["PROFILE"] = "" - if self.no_prompts: - log.info( - "Setting env var '$PROFILE' to Docker as not set.\n" - "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." - ) - os.environ["PROFILE"] = "docker" - else: - question = { - "type": "list", - "name": "profile", - "message": "Choose container software to run the test with", - "choices": ["Docker", "Singularity", "Conda"], - } - answer = questionary.unsafe_prompt([question], style=nf_core.utils.nfcore_question_style) - profile = answer["profile"].lower() - os.environ["PROFILE"] = profile - - def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: - nftest_output = Text.from_ansi(nftest_out.decode()) - print(Panel(nftest_output, title="nf-test output")) - if nftest_err: - syntax = Syntax(nftest_err.decode(), "diff", theme="ansi_dark") - print(Panel(syntax, title="nf-test error")) - if "Different Snapshot:" in nftest_err.decode(): - log.error("nf-test failed due to differences in the snapshots") - # prompt to update snapshot - if self.no_prompts: - log.info("Updating snapshot") - self.update = True - else: - answer = Confirm.ask( - "[bold][blue]?[/] nf-test found differences in the snapshot. Do you want to update it?", - default=True, - ) - if answer: - log.info("Updating snapshot") - self.update = True - else: - log.debug("Snapshot not updated") - if self.update: - # update snapshot using nf-test --update-snapshot - self.generate_snapshot() - - else: - self.errors.append("nf-test failed") - - def generate_snapshot(self) -> bool: - """Generate the nf-test snapshot using `nf-test test` command - - returns True if the test was successful, False otherwise - """ - - log.debug("Running nf-test test") - - # set verbose flag if self.verbose is True - verbose = "--verbose --debug" if self.verbose else "" - update = "--update-snapshot" if self.update else "" - self.update = False # reset self.update to False to test if the new snapshot is stable - - result = nf_core.utils.run_cmd( - "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} {verbose} {update}", - ) - if result is not None: - nftest_out, nftest_err = result - self.display_nftest_output(nftest_out, nftest_err) - # check if nftest_out contains obsolete snapshots - pattern = r"Snapshot Summary:.*?(\d+)\s+obsolete" - compiled_pattern = re.compile(pattern, re.DOTALL) # re.DOTALL to allow . to match newlines - obsolete_snapshots = compiled_pattern.search(nftest_out.decode()) - if obsolete_snapshots: - self.obsolete_snapshots = True - - # check if nf-test was successful - if "Assertion failed:" in nftest_out.decode(): - return False - elif "no valid tests found." in nftest_out.decode(): - log.error("Test file 'main.nf.test' not found") - self.errors.append("Test file 'main.nf.test' not found") - return False - else: - log.debug("nf-test successful") - return True - else: - log.error("nf-test failed") - self.errors.append("nf-test failed") - return False - - def check_snapshot_stability(self) -> bool: - """Run the nf-test twice and check if the snapshot changes""" - log.info("Generating nf-test snapshot") - if not self.generate_snapshot(): - return False # stop here if the first run failed - log.info("Generating nf-test snapshot again to check stability") - if not self.generate_snapshot(): - log.error("nf-test snapshot is not stable") - self.errors.append("nf-test snapshot is not stable") - return False - else: - if self.obsolete_snapshots: - # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot - if self.no_prompts: - log.info("Removing obsolete snapshots") - nf_core.utils.run_cmd( - "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", - ) - else: - answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) - if answer: - log.info("Removing obsolete snapshots") - nf_core.utils.run_cmd( - "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", - ) - else: - log.debug("Obsolete snapshots not removed") - return True diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index 461d813e83..4b36f302bd 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -6,7 +6,6 @@ from .list import ModuleList from .modules_json import ModulesJson from .modules_repo import ModulesRepo -from .modules_test import ModulesTest from .modules_utils import ModuleException from .patch import ModulePatch from .remove import ModuleRemove diff --git a/nf_core/modules/modules_test.py b/nf_core/modules/modules_test.py deleted file mode 100644 index d1f47dcff9..0000000000 --- a/nf_core/modules/modules_test.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -The ModulesTest class runs the tests locally -""" - -from nf_core.components.components_test import ComponentsTest - - -class ModulesTest(ComponentsTest): - """ - Class to run module pytests. - """ - - def __init__( - self, - module_name=None, - no_prompts=False, - pytest_args="", - remote_url=None, - branch=None, - no_pull=False, - ): - super().__init__( - component_type="modules", - component_name=module_name, - no_prompts=no_prompts, - pytest_args=pytest_args, - remote_url=remote_url, - branch=branch, - no_pull=no_pull, - ) diff --git a/nf_core/subworkflows/__init__.py b/nf_core/subworkflows/__init__.py index 825af5ecf5..88e8a09388 100644 --- a/nf_core/subworkflows/__init__.py +++ b/nf_core/subworkflows/__init__.py @@ -4,5 +4,4 @@ from .lint import SubworkflowLint from .list import SubworkflowList from .remove import SubworkflowRemove -from .subworkflows_test import SubworkflowsTest from .update import SubworkflowUpdate diff --git a/nf_core/subworkflows/subworkflows_test.py b/nf_core/subworkflows/subworkflows_test.py deleted file mode 100644 index d072ff678a..0000000000 --- a/nf_core/subworkflows/subworkflows_test.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -""" -The SubworkflowsTest class runs the tests locally -""" - -from nf_core.components.components_test import ComponentsTest - - -class SubworkflowsTest(ComponentsTest): - """ - Class to run module pytests. - """ - - def __init__( - self, - subworkflow_name=None, - no_prompts=False, - pytest_args="", - remote_url=None, - branch=None, - no_pull=False, - ): - super().__init__( - component_type="subworkflows", - component_name=subworkflow_name, - no_prompts=no_prompts, - pytest_args=pytest_args, - remote_url=remote_url, - branch=branch, - no_pull=no_pull, - ) diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index a18738da06..cb592065d2 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -1,10 +1,10 @@ +"""Test generate a snapshot""" import json -import os from pathlib import Path import pytest -from nf_core.components.snapshot_generator import ComponentTestSnapshotGenerator +from nf_core.components.components_test import ComponentsTest from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL, set_wd @@ -12,7 +12,7 @@ def test_generate_snapshot_module(self): """Generate the snapshot for a module in nf-core/modules clone""" with set_wd(self.nfcore_modules): - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="modules", component_name="fastqc", no_prompts=True, @@ -34,7 +34,7 @@ def test_generate_snapshot_module(self): def test_generate_snapshot_subworkflow(self): """Generate the snapshot for a subworkflows in nf-core/modules clone""" with set_wd(self.nfcore_modules): - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="subworkflows", component_name="bam_sort_stats_samtools", no_prompts=True, @@ -72,7 +72,7 @@ def test_update_snapshot_module(self): snap_content["Single-End"]["content"][0]["0"][0][1] = "" with open(snap_path, "w") as fh: json.dump(snap_content, fh) - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="modules", component_name="bwa/mem", no_prompts=True, @@ -91,7 +91,7 @@ def test_update_snapshot_module(self): def test_test_not_found(self): """Generate the snapshot for a module in nf-core/modules clone which doesn't contain tests""" with set_wd(self.nfcore_modules): - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="modules", component_name="fastp", no_prompts=True, @@ -106,7 +106,7 @@ def test_test_not_found(self): def test_unstable_snapshot(self): """Generate the snapshot for a module in nf-core/modules clone with unstable snapshots""" with set_wd(self.nfcore_modules): - snap_generator = ComponentTestSnapshotGenerator( + snap_generator = ComponentsTest( component_type="modules", component_name="kallisto/quant", no_prompts=True, diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py new file mode 100644 index 0000000000..cf07d1d67b --- /dev/null +++ b/tests/components/snapshot_test.py @@ -0,0 +1,40 @@ +"""Test the 'modules test' or 'subworkflows test' command which runs nf-test test.""" +import shutil +from pathlib import Path + +import pytest + +from nf_core.components.components_test import ComponentsTest + +from ..utils import set_wd + + +def test_components_test_check_inputs(self): + """Test the check_inputs() function - raise UserWarning because module doesn't exist""" + with set_wd(self.nfcore_modules): + meta_builder = ComponentsTest(component_type="modules", component_name="none", no_prompts=True) + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + assert "Cannot find directory" in str(excinfo.value) + + +def test_components_test_no_name_no_prompts(self): + """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" + with set_wd(self.nfcore_modules): + meta_builder = ComponentsTest(component_type="modules", component_name=None, no_prompts=True) + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + assert "Module name not provided and prompts deactivated." in str(excinfo.value) + + +def test_components_test_no_installed_modules(self): + """Test the check_inputs() function - raise UserWarning because installed modules were not found""" + with set_wd(self.nfcore_modules): + module_dir = Path(self.nfcore_modules, "modules") + shutil.rmtree(module_dir) + module_dir.mkdir() + meta_builder = ComponentsTest(component_type="modules", component_name=None, no_prompts=True) + meta_builder.repo_type = "modules" + with pytest.raises(UserWarning) as excinfo: + meta_builder.check_inputs() + assert "No installed modules were found" in str(excinfo.value) diff --git a/tests/subworkflows/subworkflows_test.py b/tests/subworkflows/subworkflows_test.py index adb0989b33..bdfb2f465f 100644 --- a/tests/subworkflows/subworkflows_test.py +++ b/tests/subworkflows/subworkflows_test.py @@ -10,24 +10,6 @@ from ..utils import set_wd -def test_subworkflows_test_check_inputs(self): - """Test the check_inputs() function - raise UserWarning because module doesn't exist""" - with set_wd(self.nfcore_modules): - meta_builder = nf_core.subworkflows.SubworkflowsTest("none", True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "Cannot find directory" in str(excinfo.value) - - -def test_subworkflows_test_no_name_no_prompts(self): - """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" - with set_wd(self.nfcore_modules): - meta_builder = nf_core.subworkflows.SubworkflowsTest(None, True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "Subworkflow name not provided and prompts deactivated." in str(excinfo.value) - - def test_subworkflows_test_no_installed_subworkflows(self): """Test the check_inputs() function - raise UserWarning because installed modules were not found""" with set_wd(self.nfcore_modules): diff --git a/tests/test_modules.py b/tests/test_modules.py index a6ae779f18..39c600b986 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -199,11 +199,6 @@ def test_modulesrepo_class(self): test_mod_json_with_empty_modules_value, test_mod_json_with_missing_modules_entry, ) - from .modules.modules_test import ( # type: ignore[misc] - test_modules_test_check_inputs, - test_modules_test_no_installed_modules, - test_modules_test_no_name_no_prompts, - ) from .modules.patch import ( # type: ignore[misc] test_create_patch_change, test_create_patch_no_change, diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 925e70fb76..33cad81e3d 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -138,11 +138,6 @@ def tearDown(self): test_subworkflows_remove_subworkflow, test_subworkflows_remove_subworkflow_keep_installed_module, ) - from .subworkflows.subworkflows_test import ( # type: ignore[misc] - test_subworkflows_test_check_inputs, - test_subworkflows_test_no_installed_subworkflows, - test_subworkflows_test_no_name_no_prompts, - ) from .subworkflows.update import ( # type: ignore[misc] test_install_and_update, test_install_at_hash_and_update, From 300289ee7944bf680169f068f1722402d0afa331 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Nov 2023 12:44:30 +0100 Subject: [PATCH 095/158] move modules and subworkflows test tests to components --- nf_core/components/components_test.py | 18 ++++++++--- tests/components/snapshot_test.py | 6 ++-- tests/modules/modules_test.py | 41 ------------------------- tests/subworkflows/subworkflows_test.py | 23 -------------- tests/test_components.py | 5 +++ 5 files changed, 21 insertions(+), 72 deletions(-) delete mode 100644 tests/modules/modules_test.py delete mode 100644 tests/subworkflows/subworkflows_test.py diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 3470fbe4a1..44ce6cd20f 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -109,11 +109,19 @@ def check_inputs(self) -> None: # Get the component name if not specified if self.component_name is None: - self.component_name = questionary.autocomplete( - "Tool name:" if self.component_type == "modules" else "Subworkflow name:", - choices=self.components_from_repo(self.org), - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() + if self.no_prompts: + raise UserWarning( + f"{self.component_type[:-1].title()} name not provided and prompts deactivated. Please provide the {self.component_type[:-1]} name{' as TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}." + ) + else: + try: + self.component_name = questionary.autocomplete( + "Tool name:" if self.component_type == "modules" else "Subworkflow name:", + choices=self.components_from_repo(self.org), + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + except LookupError: + raise self.component_dir = Path(self.component_type, self.modules_repo.repo_path, *self.component_name.split("/")) diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py index cf07d1d67b..371f0d6fbe 100644 --- a/tests/components/snapshot_test.py +++ b/tests/components/snapshot_test.py @@ -33,8 +33,8 @@ def test_components_test_no_installed_modules(self): module_dir = Path(self.nfcore_modules, "modules") shutil.rmtree(module_dir) module_dir.mkdir() - meta_builder = ComponentsTest(component_type="modules", component_name=None, no_prompts=True) + meta_builder = ComponentsTest(component_type="modules", component_name=None, no_prompts=False) meta_builder.repo_type = "modules" - with pytest.raises(UserWarning) as excinfo: + with pytest.raises(LookupError) as excinfo: meta_builder.check_inputs() - assert "No installed modules were found" in str(excinfo.value) + assert "Nothing installed from" in str(excinfo.value) diff --git a/tests/modules/modules_test.py b/tests/modules/modules_test.py deleted file mode 100644 index eb207fa28b..0000000000 --- a/tests/modules/modules_test.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Test the 'modules test' command which runs module pytests.""" -import os -import shutil -from pathlib import Path - -import pytest - -import nf_core.modules - -from ..utils import set_wd - - -def test_modules_test_check_inputs(self): - """Test the check_inputs() function - raise UserWarning because module doesn't exist""" - with set_wd(self.nfcore_modules): - meta_builder = nf_core.modules.ModulesTest("none", True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "Cannot find directory" in str(excinfo.value) - - -def test_modules_test_no_name_no_prompts(self): - """Test the check_inputs() function - raise UserWarning prompts are deactivated and module name is not provided.""" - with set_wd(self.nfcore_modules): - meta_builder = nf_core.modules.ModulesTest(None, True, "") - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "Module name not provided and prompts deactivated." in str(excinfo.value) - - -def test_modules_test_no_installed_modules(self): - """Test the check_inputs() function - raise UserWarning because installed modules were not found""" - with set_wd(self.nfcore_modules): - module_dir = Path(self.nfcore_modules, "modules") - shutil.rmtree(module_dir) - module_dir.mkdir() - meta_builder = nf_core.modules.ModulesTest(None, False, "") - meta_builder.repo_type = "modules" - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "No installed modules were found" in str(excinfo.value) diff --git a/tests/subworkflows/subworkflows_test.py b/tests/subworkflows/subworkflows_test.py deleted file mode 100644 index bdfb2f465f..0000000000 --- a/tests/subworkflows/subworkflows_test.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Test the 'subworkflows test' command which runs module pytests.""" -import os -import shutil -from pathlib import Path - -import pytest - -import nf_core.subworkflows - -from ..utils import set_wd - - -def test_subworkflows_test_no_installed_subworkflows(self): - """Test the check_inputs() function - raise UserWarning because installed modules were not found""" - with set_wd(self.nfcore_modules): - module_dir = Path(self.nfcore_modules, "subworkflows") - shutil.rmtree(module_dir) - module_dir.mkdir() - meta_builder = nf_core.subworkflows.SubworkflowsTest(None, False, "") - meta_builder.repo_type = "modules" - with pytest.raises(UserWarning) as excinfo: - meta_builder._check_inputs() - assert "No installed subworkflows were found" in str(excinfo.value) diff --git a/tests/test_components.py b/tests/test_components.py index d2b0c4630f..97db6f78bd 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -46,3 +46,8 @@ def tearDown(self): test_unstable_snapshot, test_update_snapshot_module, ) + from .components.snapshot_test import ( # type: ignore[misc] + test_components_test_check_inputs, + test_components_test_no_installed_modules, + test_components_test_no_name_no_prompts, + ) From c25469998c8dbceb1dec685094ffac1caa86d59d Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 20 Nov 2023 13:39:55 +0100 Subject: [PATCH 096/158] add test for generate snapshot once --- nf_core/__main__.py | 8 ++++++-- nf_core/components/components_command.py | 4 +++- nf_core/components/components_test.py | 11 ++++++++--- tests/components/create_snapshot.py | 20 ++++++++++++++++++++ tests/test_components.py | 1 + 5 files changed, 38 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 7e82b08990..67449ba4f9 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -859,7 +859,8 @@ def create_module( @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") -def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update): +@click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") +def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update, once): """ Generate nf-test snapshots for a module. @@ -875,6 +876,7 @@ def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update): run_tests=run_tests, no_prompts=no_prompts, update=update, + once=once, remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], @@ -1066,7 +1068,8 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") -def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update): +@click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") +def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update, once): """ Generate nf-test snapshots for a module. Given the name of a module, runs the nf-test command to generate snapshots. @@ -1081,6 +1084,7 @@ def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update): run_tests=run_tests, no_prompts=no_prompts, update=update, + once=once, remote_url=ctx.obj["modules_repo_url"], branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index fdfebde735..44924a2704 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -27,6 +27,7 @@ def __init__( branch: Optional[str] = None, no_pull: bool = False, hide_progress: bool = False, + no_prompts: bool = False, ) -> None: """ Initialise the ComponentClass object @@ -35,6 +36,7 @@ def __init__( self.dir = dir self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress) self.hide_progress = hide_progress + self.no_prompts = no_prompts self._configure_repo_and_paths() def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: @@ -48,7 +50,7 @@ def _configure_repo_and_paths(self, nf_dir_req: bool = True) -> None: try: if self.dir: - self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req) + self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=not self.no_prompts) else: self.repo_type = None self.org = "" diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 44ce6cd20f..24bc7cfe43 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -50,6 +50,8 @@ class ComponentsTest(ComponentCommand): flag indicating if verbose output should be used update : bool flag indicating if the existing snapshot should be updated + once : bool + flag indicating if the test should be run only once Methods ------- @@ -74,17 +76,18 @@ def __init__( branch: Optional[str] = None, verbose: bool = False, update: bool = False, + once: bool = False, ): - super().__init__(component_type, directory, remote_url, branch) + super().__init__(component_type, directory, remote_url, branch, no_prompts=no_prompts) self.component_name = component_name self.remote_url = remote_url self.branch = branch self.run_tests = run_tests - self.no_prompts = no_prompts self.errors: List[str] = [] self.verbose = verbose self.obsolete_snapshots: bool = False self.update = update + self.once = once def run(self) -> None: """Run build steps""" @@ -168,7 +171,7 @@ def display_nftest_output(self, nftest_out: bytes, nftest_err: bytes) -> None: if self.no_prompts: log.info("Updating snapshot") self.update = True - else: + elif self.update is None: answer = Confirm.ask( "[bold][blue]?[/] nf-test found differences in the snapshot. Do you want to update it?", default=True, @@ -232,6 +235,8 @@ def check_snapshot_stability(self) -> bool: log.info("Generating nf-test snapshot") if not self.generate_snapshot(): return False # stop here if the first run failed + elif self.once: + return True # stop here if the test should be run only once log.info("Generating nf-test snapshot again to check stability") if not self.generate_snapshot(): log.error("nf-test snapshot is not stable") diff --git a/tests/components/create_snapshot.py b/tests/components/create_snapshot.py index cb592065d2..c7eb696722 100644 --- a/tests/components/create_snapshot.py +++ b/tests/components/create_snapshot.py @@ -1,6 +1,7 @@ """Test generate a snapshot""" import json from pathlib import Path +from unittest.mock import MagicMock import pytest @@ -60,6 +61,25 @@ def test_generate_snapshot_subworkflow(self): ) +def test_generate_snapshot_once( + self, +): + """Generate the snapshot for a module in nf-core/modules clone only once""" + with set_wd(self.nfcore_modules): + snap_generator = ComponentsTest( + component_type="modules", + component_name="fastqc", + once=True, + no_prompts=True, + remote_url=GITLAB_URL, + branch=GITLAB_NFTEST_BRANCH, + ) + snap_generator.repo_type = "modules" + snap_generator.generate_snapshot = MagicMock() + snap_generator.run() + snap_generator.generate_snapshot.assert_called_once() + + def test_update_snapshot_module(self): """Update the snapshot of a module in nf-core/modules clone""" diff --git a/tests/test_components.py b/tests/test_components.py index 97db6f78bd..8a28d4cbcc 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -41,6 +41,7 @@ def tearDown(self): from .components.create_snapshot import ( # type: ignore[misc] test_generate_snapshot_module, + test_generate_snapshot_once, test_generate_snapshot_subworkflow, test_test_not_found, test_unstable_snapshot, From bb640bb7cf7baebf8a312f64ea866f4a3cf3f4a5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 15:39:01 +0100 Subject: [PATCH 097/158] update README --- README.md | 76 ++++++++++++++++++++----------------------------------- 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index c9bcd25398..28e17efb0b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ A python package with helper tools for the nf-core community. - [`modules remove` - Remove a module from a pipeline](#remove-a-module-from-a-pipeline) - [`modules patch` - Create a patch file for a module](#create-a-patch-file-for-a-module) - [`modules create` - Create a module from the template](#create-a-new-module) - - [`modules create-test-yml` - Create the `test.yml` file for a module](#create-a-module-test-config-file) - [`modules lint` - Check a module against nf-core guidelines](#check-a-module-against-nf-core-guidelines) - [`modules test` - Run the tests for a module](#run-the-tests-for-a-module-using-pytest) - [`modules bump-versions` - Bump software versions of modules](#bump-bioconda-and-container-versions-of-modules-in) @@ -53,7 +52,6 @@ A python package with helper tools for the nf-core community. - [`subworkflows update` - Update subworkflows in a pipeline](#update-subworkflows-in-a-pipeline) - [`subworkflows remove` - Remove a subworkflow from a pipeline](#remove-a-subworkflow-from-a-pipeline) - [`subworkflows create` - Create a subworkflow from the template](#create-a-new-subworkflow) - - [`subworkflows create-test-yml` - Create the `test.yml` file for a subworkflow](#create-a-subworkflow-test-config-file) - [`subworkflows lint` - Check a subworkflow against nf-core guidelines](#check-a-subworkflow-against-nf-core-guidelines) - [`subworkflows test` - Run the tests for a subworkflow](#run-the-tests-for-a-subworkflow-using-pytest) - [Citation](#citation) @@ -947,20 +945,6 @@ fake_command: nf-core modules create fastqc --author @nf-core-bot --label proce ![`cd modules && nf-core modules create fastqc --author @nf-core-bot --label process_low --meta --force`](docs/images/nf-core-modules-create.svg) -### Create a module test config file - -All modules on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. -To help developers build new modules, the `nf-core modules create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. -After you have written a minimal Nextflow script to test your module `tests/modules///main.nf`, this command will run the tests for you and create the `tests/modules///test.yml` file. - - - -![`nf-core modules create-test-yml fastqc --no-prompts --force`](docs/images/nf-core-modules-create-test.svg) - ### Check a module against nf-core guidelines Run the `nf-core modules lint` command to check modules in the current working directory (pipeline or nf-core/modules clone) against nf-core guidelines. @@ -974,15 +958,12 @@ before_command: sed 's/1.13a/1.10/g' modules/multiqc/main.nf > modules/multiqc/m ![`nf-core modules lint multiqc`](docs/images/nf-core-modules-lint.svg) -### Run the tests for a module using pytest - -To run unit tests of a module that you have installed or the test created by the command [`nf-core modules create-test-yml`](#create-a-module-test-config-file), you can use `nf-core modules test` command. This command runs the tests specified in `modules/tests/software///test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). +### Create a test for a module -:::info -This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new module to add the new files to your git index. -::: +All modules on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. We use [nf-test](https://code.askimed.com/nf-test/) as our testing framework. +Each module coomes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core modules test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. -You can specify the module name in the form TOOL/SUBTOOL in command line or provide it later by prompts. +You can specify the module name in the form TOOL/SUBTOOL in the command or provide it later through interactive prompts. -![`nf-core modules test samtools/view --no-prompts`](docs/images/nf-core-modules-test.svg) +![`nf-core modules test fastqc --no-prompts --force`](docs/images/nf-core-modules-test.svg) + +In case you changed something in the test and want to update the snapshot, run + +```bash +nf-core modules test --update +``` + +If you want to run the test only once without checking for snapshot stability, you can use the `--once` flag. ### Bump bioconda and container versions of modules in @@ -1221,19 +1210,29 @@ fake_command: nf-core subworkflows create bam_stats_samtools --author @nf-core-b ![`nf-core subworkflows create bam_stats_samtools --author @nf-core-bot --force`](docs/images/nf-core-subworkflows-create.svg) -### Create a subworkflow test config file +### Create a test for a subworkflow + +All subworkflows on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. We use [nf-test](https://code.askimed.com/nf-test/) as our testing framework. +Each subworkflow coomes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core subworkflows test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. -All subworkflows on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. -To help developers build new subworkflows, the `nf-core subworkflows create-test-yml` command automates the creation of the yaml file required to document the output file `md5sum` and other information generated by the testing. -After you have written a minimal Nextflow script to test your subworkflow in `/tests/subworkflow//main.nf`, this command will run the tests for you and create the `/tests/subworkflow///test.yml` file. +You can specify the subworkflow name in the command or provide it later through interactive prompts. -![`nf-core subworkflows create-test-yml bam_stats_samtools --no-prompts --force`](docs/images/nf-core-subworkflows-create-test.svg) +![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) + +In case you changed something in the test and want to update the snapshot, run + +```bash +nf-core modules test --update +``` + +If you want to run the test only once without checking for snapshot stability, you can use the `--once` flag. ### Check a subworkflow against nf-core guidelines @@ -1249,25 +1248,6 @@ extra_env: ![`nf-core subworkflows lint bam_stats_samtools`](docs/images/nf-core-subworkflows-lint.svg) -### Run the tests for a subworkflow using pytest - -To run unit tests of a subworkflow that you have installed or the test created by the command [`nf-core subworkflow create-test-yml`](#create-a-subworkflow-test-config-file), you can use `nf-core subworkflows test` command. This command runs the tests specified in `tests/subworkflows//test.yml` file using [pytest](https://pytest-workflow.readthedocs.io/en/stable/). - -:::info -This command uses the pytest argument `--git-aware` to avoid copying the whole `.git` directory and files ignored by `git`. This means that it will only include files listed by `git ls-files`. Remember to **commit your changes** after adding a new subworkflow to add the new files to your git index. -::: - -You can specify the subworkflow name in the form TOOL/SUBTOOL in command line or provide it later by prompts. - - - -![`nf-core subworkflows test bam_rseqc --no-prompts`](docs/images/nf-core-subworkflows-test.svg) - ## Citation If you use `nf-core tools` in your work, please cite the `nf-core` publication as follows: From c550823b2c7390af58ca9de4247962463d68ef88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 20 Nov 2023 15:44:53 +0100 Subject: [PATCH 098/158] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 28e17efb0b..8e6b52d4c2 100644 --- a/README.md +++ b/README.md @@ -961,7 +961,7 @@ before_command: sed 's/1.13a/1.10/g' modules/multiqc/main.nf > modules/multiqc/m ### Create a test for a module All modules on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. We use [nf-test](https://code.askimed.com/nf-test/) as our testing framework. -Each module coomes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core modules test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. +Each module comes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core modules test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. You can specify the module name in the form TOOL/SUBTOOL in the command or provide it later through interactive prompts. @@ -1213,7 +1213,7 @@ fake_command: nf-core subworkflows create bam_stats_samtools --author @nf-core-b ### Create a test for a subworkflow All subworkflows on [nf-core/modules](https://github.com/nf-core/modules) have a strict requirement of being unit tested using minimal test data. We use [nf-test](https://code.askimed.com/nf-test/) as our testing framework. -Each subworkflow coomes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core subworkflows test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. +Each subworkflow comes already with a template for the test file in `test/main.nf.test`. Replace the placeholder code in that file with your specific input, output and proces. In order to generate the corresponding snapshot after writing your test, you can use the `nf-core subworkflows test` command. This command will run `nf-test test` twice, to also check for snapshot stability, i.e. that the same snapshot is generated on multiple runs. You can specify the subworkflow name in the command or provide it later through interactive prompts. @@ -1229,7 +1229,7 @@ extra_env: In case you changed something in the test and want to update the snapshot, run ```bash -nf-core modules test --update +nf-core subworkflows test --update ``` If you want to run the test only once without checking for snapshot stability, you can use the `--once` flag. From 2b15177f1a112ef063a1afd28ce09a27656172f7 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 16:50:07 +0100 Subject: [PATCH 099/158] fix typing errors --- nf_core/components/components_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 24bc7cfe43..36a1cbaa3c 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -24,7 +24,7 @@ log = logging.getLogger(__name__) -class ComponentsTest(ComponentCommand): +class ComponentsTest(ComponentCommand): # type: ignore[misc] """ Class to generate and test nf-test snapshots for modules. @@ -96,7 +96,7 @@ def run(self) -> None: os.environ[ "NFT_DIFF_ARGS" ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences - with set_wd(self.dir): + with set_wd(Path(self.dir)): self.check_snapshot_stability() if len(self.errors) > 0: errors = "\n - ".join(self.errors) From f4a48b56c006d00225a6f8a0dc60636907a1ece9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 17:47:46 +0100 Subject: [PATCH 100/158] remove last traces of create-snapshot or create-test-yml --- nf_core/__main__.py | 4 ++-- nf_core/module-template/modules/tests/main.nf.test | 2 +- nf_core/subworkflow-template/subworkflows/tests/main.nf.test | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 67449ba4f9..979931df6d 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -44,7 +44,7 @@ }, { "name": "Developing new modules", - "commands": ["create", "create-test-yml", "lint", "bump-versions", "test"], + "commands": ["create", "lint", "bump-versions", "test"], }, ], "nf-core subworkflows": [ @@ -54,7 +54,7 @@ }, { "name": "Developing new subworkflows", - "commands": ["create", "create-test-yml"], + "commands": ["create", "test", "lint"], }, ], } diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index 0e08f89e87..883a3ffa40 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -1,5 +1,5 @@ // TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core modules create-nf-test {{ component_name }} +// nf-core modules test {{ component_name }} nextflow_process { name "Test Process {{ component_name_underscore|upper }}" diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index a0a18e0739..b0a212f2a3 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -1,5 +1,5 @@ // TODO nf-core: Once you have added the required tests, please run the following command to build this file: -// nf-core subworkflows create-nf-test {{ component_name }} +// nf-core subworkflows test {{ component_name }} nextflow_workflow { name "Test Workflow {{ component_name_underscore|upper }}" From 9f2f90576a703deadec127ce827ce7077d201e89 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 17:51:44 +0100 Subject: [PATCH 101/158] fix parameters --- nf_core/__main__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 979931df6d..4f80176e24 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -666,6 +666,7 @@ def install(ctx, tool, dir, prompt, force, sha): @click.option( "-d", "--dir", + "directory", type=click.Path(exists=True), default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", @@ -698,7 +699,7 @@ def install(ctx, tool, dir, prompt, force, sha): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, tool, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): +def update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 modules within a pipeline. @@ -708,11 +709,11 @@ def update(ctx, tool, dir, force, prompt, sha, install_all, preview, save_diff, try: module_install = ModuleUpdate( - dir, + directory, force, prompt, sha, - all, + install_all, preview, save_diff, update_deps, @@ -1307,7 +1308,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): ctx.obj["modules_repo_no_pull"], ) exit_status = subworkflow_install.install(subworkflow) - if not exit_status and install_all: + if not exit_status: sys.exit(1) except (UserWarning, LookupError) as e: log.error(e) From f2885204daf6be4d6dff2eb682ebf956c364df65 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 18:32:16 +0100 Subject: [PATCH 102/158] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 994c17901a..b8fcce59fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ ### General +- Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] From 7722d521c0aae70d67fc7cfd78bf1c4ba8aa058c Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 20 Nov 2023 18:55:31 +0100 Subject: [PATCH 103/158] rename create_snapshot functions and files --- nf_core/__main__.py | 4 ++-- .../components/{create_snapshot.py => generate_snapshot.py} | 0 tests/test_components.py | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) rename tests/components/{create_snapshot.py => generate_snapshot.py} (100%) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4f80176e24..d0677b3fde 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -861,7 +861,7 @@ def create_module( @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") -def create_snapshot(ctx, tool, dir, run_tests, no_prompts, update, once): +def test_module(ctx, tool, dir, run_tests, no_prompts, update, once): """ Generate nf-test snapshots for a module. @@ -1070,7 +1070,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") -def create_snapshot(ctx, subworkflow, dir, run_tests, no_prompts, update, once): +def test_subworkflow(ctx, subworkflow, dir, run_tests, no_prompts, update, once): """ Generate nf-test snapshots for a module. Given the name of a module, runs the nf-test command to generate snapshots. diff --git a/tests/components/create_snapshot.py b/tests/components/generate_snapshot.py similarity index 100% rename from tests/components/create_snapshot.py rename to tests/components/generate_snapshot.py diff --git a/tests/test_components.py b/tests/test_components.py index 8a28d4cbcc..b7f67eb51d 100644 --- a/tests/test_components.py +++ b/tests/test_components.py @@ -7,9 +7,7 @@ import unittest from pathlib import Path -from git import Repo - -from nf_core.modules.modules_repo import ModulesRepo +from git.repo import Repo from .utils import GITLAB_NFTEST_BRANCH, GITLAB_URL @@ -39,7 +37,7 @@ def tearDown(self): # Test of the individual components commands. # ############################################ - from .components.create_snapshot import ( # type: ignore[misc] + from .components.generate_snapshot import ( # type: ignore[misc] test_generate_snapshot_module, test_generate_snapshot_once, test_generate_snapshot_subworkflow, From 71c0b08714e035ac87889cb0bc5a65c93f2a1bd8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 09:17:42 +0100 Subject: [PATCH 104/158] simplify obsolete check --- nf_core/components/components_test.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 36a1cbaa3c..f7de592c77 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -245,20 +245,14 @@ def check_snapshot_stability(self) -> bool: else: if self.obsolete_snapshots: # ask if the user wants to remove obsolete snapshots using nf-test --clean-snapshot - if self.no_prompts: + if self.no_prompts or Confirm.ask( + "nf-test found obsolete snapshots. Do you want to remove them?", default=True + ): log.info("Removing obsolete snapshots") nf_core.utils.run_cmd( "nf-test", f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", ) else: - answer = Confirm.ask("nf-test found obsolete snapshots. Do you want to remove them?", default=True) - if answer: - log.info("Removing obsolete snapshots") - nf_core.utils.run_cmd( - "nf-test", - f"test --tag {self.component_name} --profile {os.environ['PROFILE']} --clean-snapshot", - ) - else: - log.debug("Obsolete snapshots not removed") + log.debug("Obsolete snapshots not removed") return True From d29643eb5e26e55b45eb101991f981bd89befb6b Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 09:28:57 +0100 Subject: [PATCH 105/158] add back option to handle pytest based modules in bump_version --- nf_core/modules/bump_versions.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 45bceba131..0a70ea711b 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -136,8 +136,21 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: module: NFCoreComponent """ config_version = None - # Extract bioconda version from `main.nf` - bioconda_packages = self.get_bioconda_version(module) + bioconda_packages = [] + try: + # Extract bioconda version from `environment.yml` + bioconda_packages = self.get_bioconda_version(module) + except FileNotFoundError: + # try it in the main.nf instead + try: + with open(module.main_nf, "r") as fh: + for l in fh: + if "bioconda::" in l: + bioconda_packages = [b for b in l.split() if "bioconda::" in b] + except FileNotFoundError: + log.error( + f"Neither `environment.yml` nor `main.nf` of {module.component_name} module could be read to get bioconada version of used tools." + ) # If multiple versions - don't update! (can't update mulled containers) if not bioconda_packages or len(bioconda_packages) > 1: From 4d08f6d829145ef1ae4c9225782300a082b326d8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:04:50 +0100 Subject: [PATCH 106/158] rename generator to test to make it hopefully less confusing --- nf_core/__main__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d0677b3fde..5833066bac 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -863,14 +863,14 @@ def create_module( @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") def test_module(ctx, tool, dir, run_tests, no_prompts, update, once): """ - Generate nf-test snapshots for a module. + Run nf-test for a module. - Given the name of a module, runs the nf-test command to generate snapshots. + Given the name of a module, runs the nf-test command to test the module and generate snapshots. """ from nf_core.components.components_test import ComponentsTest try: - snap_generator = ComponentsTest( + module_tester = ComponentsTest( component_type="modules", component_name=tool, directory=dir, @@ -882,7 +882,7 @@ def test_module(ctx, tool, dir, run_tests, no_prompts, update, once): branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], ) - snap_generator.run() + module_tester.run() except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) @@ -1072,13 +1072,14 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") def test_subworkflow(ctx, subworkflow, dir, run_tests, no_prompts, update, once): """ - Generate nf-test snapshots for a module. - Given the name of a module, runs the nf-test command to generate snapshots. + Run nf-test for a subworkflow. + + Given the name of a subworkflow, runs the nf-test command to test the subworkflow and generate snapshots. """ from nf_core.components.components_test import ComponentsTest try: - snap_generator = ComponentsTest( + sw_tester = ComponentsTest( component_type="subworkflows", component_name=subworkflow, directory=dir, @@ -1090,7 +1091,7 @@ def test_subworkflow(ctx, subworkflow, dir, run_tests, no_prompts, update, once) branch=ctx.obj["modules_repo_branch"], verbose=ctx.obj["verbose"], ) - snap_generator.run() + sw_tester.run() except (UserWarning, LookupError) as e: log.critical(e) sys.exit(1) From 0c120abc536d915c2519a292c17e45560f30c068 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:05:16 +0100 Subject: [PATCH 107/158] automatically sort dependencies in environment.yml while linting --- nf_core/modules/lint/environment_yml.py | 21 +++++++++++++-------- tests/modules/lint.py | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 9983ab1c31..65fe4207ad 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -1,12 +1,16 @@ import json +import logging from pathlib import Path import yaml from jsonschema import exceptions, validators +from utils import custom_yaml_dumper from nf_core.components.lint import ComponentLint from nf_core.components.nfcore_component import NFCoreComponent +log = logging.getLogger(__name__) + def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None: """ @@ -74,18 +78,19 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.passed.append( ( "environment_yml_sorted", - "Module's `environment.yml` is sorted alphabetically", + "The dependencies in the module's `environment.yml` were not sorted alphabetically", module.environment_yml, ) ) else: - module.failed.append( - ( - "environment_yml_sorted", - "Module's `environment.yml` is not sorted alphabetically", - module.environment_yml, - ) + # sort it and write it back to the file + log.info( + f"Dependencies in {module.component_name}'s environment.yml were not sorted alphabetically. Sorting them now." ) + env_yml["dependencies"].sort() + with open(Path(module.component_dir, "environment.yml"), "w") as fh: + yaml.dump(env_yml, fh, Dumper=custom_yaml_dumper()) + # Check that the name in the environment.yml file matches the name in the meta.yml file with open(Path(module.component_dir, "meta.yml"), "r") as fh: meta_yml = yaml.safe_load(fh) @@ -94,7 +99,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.passed.append( ( "environment_yml_name", - "Module's `environment.yml` name matches module name", + "The module's `environment.yml` name matches module name", module.environment_yml, ) ) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 8263db1260..a3a7a80b71 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -380,10 +380,10 @@ def test_modules_environment_yml_file_sorted_incorrectly(self): fh.write(yaml_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + # we fix the sorting on the fly, so this should pass + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert module_lint.failed[0].lint_test == "environment_yml_sorted" def test_modules_environment_yml_file_not_array(self): From dc5c7cb7f03131711eca6aaf09ee5d7db30cae25 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:05:54 +0100 Subject: [PATCH 108/158] fix types --- nf_core/modules/bump_versions.py | 33 +++++++++++++++----------------- nf_core/utils.py | 2 +- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 0a70ea711b..cb77d4c043 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -12,16 +12,19 @@ from typing import Any, Dict, List, Optional, Tuple, Union import questionary -import rich import yaml +from rich.box import ROUNDED from rich.console import Console from rich.markdown import Markdown +from rich.panel import Panel +from rich.progress import BarColumn, Progress from rich.table import Table import nf_core.modules.modules_utils import nf_core.utils from nf_core.components.components_command import ComponentCommand from nf_core.components.nfcore_component import NFCoreComponent +from nf_core.utils import custom_yaml_dumper from nf_core.utils import plural_s as _s from nf_core.utils import rich_force_colors @@ -109,9 +112,9 @@ def bump_versions( if len(nfcore_modules) == 0: raise nf_core.modules.modules_utils.ModuleException(f"Could not find the specified module: '{module}'") - progress_bar = rich.progress.Progress( + progress_bar = Progress( "[bold blue]{task.description}", - rich.progress.BarColumn(bar_width=None), + BarColumn(bar_width=None), "[magenta]{task.completed} of {task.total}[reset] » [bold yellow]{task.fields[test_name]}", transient=True, disable=os.environ.get("HIDE_PROGRESS", None) is not None, @@ -242,7 +245,7 @@ def bump_module_version(self, module: NFCoreComponent) -> bool: env_yml = yaml.safe_load(fh) re.sub(bioconda_packages[0], f"'bioconda::{bioconda_tool_name}={last_ver}'", env_yml["dependencies"]) with open(module.environment_yml, "w") as fh: - yaml.dump(env_yml, fh, default_flow_style=False) + yaml.dump(env_yml, fh, default_flow_style=False, Dumper=custom_yaml_dumper()) self.updated.append( ( @@ -314,12 +317,12 @@ def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: # Table of up to date modules if len(self.up_to_date) > 0 and self.show_up_to_date: console.print( - rich.panel.Panel( + Panel( rf"[!] {len(self.up_to_date)} Module{_s(self.up_to_date)} version{_s(self.up_to_date)} up to date.", style="bold green", ) ) - table = Table(style="green", box=rich.box.ROUNDED) + table = Table(style="green", box=ROUNDED) table.add_column("Module name", width=max_mod_name_len) table.add_column("Update Message") table = format_result(self.up_to_date, table) @@ -327,10 +330,8 @@ def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: # Table of updated modules if len(self.updated) > 0: - console.print( - rich.panel.Panel(rf"[!] {len(self.updated)} Module{_s(self.updated)} updated", style="bold yellow") - ) - table = Table(style="yellow", box=rich.box.ROUNDED) + console.print(Panel(rf"[!] {len(self.updated)} Module{_s(self.updated)} updated", style="bold yellow")) + table = Table(style="yellow", box=ROUNDED) table.add_column("Module name", width=max_mod_name_len) table.add_column("Update message") table = format_result(self.updated, table) @@ -338,10 +339,8 @@ def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: # Table of modules that couldn't be updated if len(self.failed) > 0: - console.print( - rich.panel.Panel(rf"[!] {len(self.failed)} Module update{_s(self.failed)} failed", style="bold red") - ) - table = Table(style="red", box=rich.box.ROUNDED) + console.print(Panel(rf"[!] {len(self.failed)} Module update{_s(self.failed)} failed", style="bold red")) + table = Table(style="red", box=ROUNDED) table.add_column("Module name", width=max_mod_name_len) table.add_column("Update message") table = format_result(self.failed, table) @@ -349,10 +348,8 @@ def format_result(module_updates: List[Tuple[str, str]], table: Table) -> Table: # Table of modules ignored due to `.nf-core.yml` if len(self.ignored) > 0: - console.print( - rich.panel.Panel(rf"[!] {len(self.ignored)} Module update{_s(self.ignored)} ignored", style="grey58") - ) - table = Table(style="grey58", box=rich.box.ROUNDED) + console.print(Panel(rf"[!] {len(self.ignored)} Module update{_s(self.ignored)} ignored", style="grey58")) + table = Table(style="grey58", box=ROUNDED) table.add_column("Module name", width=max_mod_name_len) table.add_column("Update message") table = format_result(self.ignored, table) diff --git a/nf_core/utils.py b/nf_core/utils.py index 9326ff1c93..c57990a9d3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -998,7 +998,7 @@ def get_repo_releases_branches(pipeline, wfs): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -def load_tools_config(directory="."): +def load_tools_config(directory: Union[str, Path] = "."): """ Parse the nf-core.yml configuration file From a5a3fd48b5e3b1f60f89f7130dbb5b41bade5103 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:22:16 +0100 Subject: [PATCH 109/158] fix typo --- nf_core/modules/lint/environment_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 65fe4207ad..4e994ed608 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -78,7 +78,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) module.passed.append( ( "environment_yml_sorted", - "The dependencies in the module's `environment.yml` were not sorted alphabetically", + "The dependencies in the module's `environment.yml` are sorted alphabetically", module.environment_yml, ) ) From 41b61c1bf23956ab05acfe3a5ebb96463762091c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:23:58 +0100 Subject: [PATCH 110/158] fix import statement --- nf_core/modules/lint/environment_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 4e994ed608..d9c0cbf7f9 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -4,10 +4,10 @@ import yaml from jsonschema import exceptions, validators -from utils import custom_yaml_dumper from nf_core.components.lint import ComponentLint from nf_core.components.nfcore_component import NFCoreComponent +from nf_core.utils import custom_yaml_dumper log = logging.getLogger(__name__) From d2df78ab1fcc79ead60e669b17c6d98866a0ef52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 21 Nov 2023 10:45:59 +0100 Subject: [PATCH 111/158] Update nf_core/components/components_test.py --- nf_core/components/components_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index f7de592c77..d223ad85ef 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -144,7 +144,7 @@ def check_inputs(self) -> None: os.environ["PROFILE"] = "" if self.no_prompts: log.info( - "Setting env var '$PROFILE' to Docker as not set.\n" + "Setting environment variable '$PROFILE' to Docker as not set otherwise.\n" "To use Singularity set 'export PROFILE=singularity' in your shell before running this command." ) os.environ["PROFILE"] = "docker" From 75a07c4acdca03e3e083e7d648e4890d70bb468c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 10:46:25 +0100 Subject: [PATCH 112/158] remove self.run_tests --- nf_core/__main__.py | 7 ++----- nf_core/components/components_test.py | 9 --------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5833066bac..232f1bf116 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -857,11 +857,10 @@ def create_module( @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") -@click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") -def test_module(ctx, tool, dir, run_tests, no_prompts, update, once): +def test_module(ctx, tool, dir, no_prompts, update, once): """ Run nf-test for a module. @@ -874,7 +873,6 @@ def test_module(ctx, tool, dir, run_tests, no_prompts, update, once): component_type="modules", component_name=tool, directory=dir, - run_tests=run_tests, no_prompts=no_prompts, update=update, once=once, @@ -1070,7 +1068,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") -def test_subworkflow(ctx, subworkflow, dir, run_tests, no_prompts, update, once): +def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once): """ Run nf-test for a subworkflow. @@ -1083,7 +1081,6 @@ def test_subworkflow(ctx, subworkflow, dir, run_tests, no_prompts, update, once) component_type="subworkflows", component_name=subworkflow, directory=dir, - run_tests=run_tests, no_prompts=no_prompts, update=update, once=once, diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index f7de592c77..31073beeb3 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -38,8 +38,6 @@ class ComponentsTest(ComponentCommand): # type: ignore[misc] name of the tool to run tests for directory: str path to modules repository directory - run_tests : bool - flag indicating if tests should be run no_prompts : bool flat indicating if prompts are used remote_url : str @@ -70,7 +68,6 @@ def __init__( component_type: str, component_name: Optional[str] = None, directory: str = ".", - run_tests: bool = False, no_prompts: bool = False, remote_url: Optional[str] = None, branch: Optional[str] = None, @@ -82,7 +79,6 @@ def __init__( self.component_name = component_name self.remote_url = remote_url self.branch = branch - self.run_tests = run_tests self.errors: List[str] = [] self.verbose = verbose self.obsolete_snapshots: bool = False @@ -134,11 +130,6 @@ def check_inputs(self) -> None: f"Cannot find directory '{self.component_dir}'.{' Should be TOOL/SUBTOOL or TOOL' if self.component_type == 'modules' else ''}" ) - # Check that we're running tests if no prompts - if not self.run_tests and self.no_prompts: - log.debug("Setting run_tests to True as running without prompts") - self.run_tests = True - # Check container software to use if os.environ.get("PROFILE") is None: os.environ["PROFILE"] = "" From 0bcc6d876fb261b41d828da8fe35ebeff00bbc5f Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 12:03:08 +0100 Subject: [PATCH 113/158] add mypy check --- .github/workflows/lint-code.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index edff9a8c82..776f2c4cc7 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -6,6 +6,7 @@ on: pull_request: release: types: [published] + workflow_dispatch: # Cancel if a newer run is started concurrency: @@ -125,3 +126,12 @@ jobs: - name: Run if any of the listed files above is changed if: steps.changed-py-files.outputs.any_changed == 'true' run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} + + - name: MyPy Check + uses: photonbit/mypy-imprecision-action@main + with: + max_tip: "60" + sufficient_tip: "10" + enable_threshold_check: "true" + enable_base_tip_check: "true" + github_token: ${{ secrets.GITHUB_TOKEN }} From e1f913ea7585085f641b2a6d12278ab56c129ae6 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 12:32:12 +0100 Subject: [PATCH 114/158] run full test matrix only on PRs to the master branch, update python target to 3.12 --- .github/workflows/pytest.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4d9881d6bb..0fc7f3d094 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -28,12 +28,11 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] runner: ["ubuntu-latest"] include: - runner: "ubuntu-20.04" python-version: "3.8" - steps: - uses: actions/checkout@v3 name: Check out source-code repository @@ -43,6 +42,9 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: "pip" + # run full test matrix only on PRs to the master branch + if: github.ref == 'refs/heads/master' || matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8' + - name: Install dependencies run: | python -m pip install --upgrade pip -r requirements-dev.txt From 3878199d1696847258f5e54392f511613f87be7a Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 12:48:08 +0100 Subject: [PATCH 115/158] only run mypy step for PRs against dev --- .github/workflows/lint-code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 776f2c4cc7..6756597adf 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -135,3 +135,4 @@ jobs: enable_threshold_check: "true" enable_base_tip_check: "true" github_token: ${{ secrets.GITHUB_TOKEN }} + if: ${{ github.event_name == 'pull_request' && github.head_ref == 'dev' }} From 1be57f608f4216f67a20c97b7be70f13bdfaf748 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 13:12:40 +0100 Subject: [PATCH 116/158] upgrade to python 3.12 --- .github/workflows/create-lint-wf.yml | 4 ++-- .github/workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 4 ++-- .github/workflows/deploy-pypi.yml | 4 ++-- .github/workflows/fix-linting.yml | 4 ++-- .github/workflows/lint-code.yml | 6 +++--- .github/workflows/sync.yml | 4 ++-- .github/workflows/tools-api-docs-dev.yml | 4 ++-- .github/workflows/tools-api-docs-release.yml | 4 ++-- CHANGELOG.md | 1 + README.md | 2 +- nf_core/pipeline-template/.github/workflows/linting.yml | 2 +- 12 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 8f6836f309..d5f377a486 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -31,10 +31,10 @@ jobs: name: Check out source-code repository # Set up nf-core/tools - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 2689805dd1..5ce9b41118 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -36,10 +36,10 @@ jobs: - uses: actions/checkout@v3 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 5faa59772c..d96518d7c6 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -29,10 +29,10 @@ jobs: - uses: actions/checkout@v3 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 1f539fe09a..9b46f0731b 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -16,10 +16,10 @@ jobs: - uses: actions/checkout@v3 name: Check out source-code repository - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index ed2314046a..f56cb927fc 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -38,10 +38,10 @@ jobs: # Override to remove the default --check flag so that we make changes options: "--color" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: python-isort uses: isort/isort-action@v1.0.0 with: diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index edff9a8c82..a951230a6a 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -76,10 +76,10 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: python-isort uses: isort/isort-action@v1.1.0 with: @@ -92,7 +92,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v3 with: - python-version: 3.11 + python-version: 3.12 cache: "pip" - name: Install dependencies diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index fbbdacc8ab..5e7719973f 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -48,10 +48,10 @@ jobs: path: nf-core/${{ matrix.pipeline }} fetch-depth: "0" - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index add939aba1..3a4725a944 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -26,10 +26,10 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index f049d74ca0..8a6eda1318 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -21,10 +21,10 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v4 with: - python-version: 3.11 + python-version: 3.12 - name: Install python dependencies run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index b8fcce59fb..f64047db4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) +- Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)). # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] diff --git a/README.md b/README.md index 8e6b52d4c2..9dcafe4386 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.11 nf-core nextflow +conda create --name nf-core python=3.12 nf-core nextflow conda activate nf-core ``` diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 29fc466ed6..edce89065f 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -78,7 +78,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies From fa3f6b70a8bd86f473600c71315d3c6b2b724fc0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 13:47:23 +0100 Subject: [PATCH 117/158] add manual dispatch exception to mypy check --- .github/workflows/lint-code.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index f214bc680b..b4bd2fa4fb 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -135,4 +135,5 @@ jobs: enable_threshold_check: "true" enable_base_tip_check: "true" github_token: ${{ secrets.GITHUB_TOKEN }} - if: ${{ github.event_name == 'pull_request' && github.head_ref == 'dev' }} + # only run on PRs to dev branch or manual dispatch + if: ${{ github.event_name == 'pull_request' && github.head_ref == 'dev' }} || github.event_name == 'workflow_dispatch' From 5bba99cd83f12fe2ed4a714c7ea3fa3aeb183783 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 13:56:28 +0100 Subject: [PATCH 118/158] add mypy.ini --- mypy.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000000..f869fa7e47 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +warn_unused_configs = True From c23fed6bcf6d8e625a8143914fa8f1b9c5494dfc Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 14:02:26 +0100 Subject: [PATCH 119/158] add it to the action --- .github/workflows/lint-code.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index b4bd2fa4fb..737b5f11ac 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -132,6 +132,7 @@ jobs: with: max_tip: "60" sufficient_tip: "10" + mypy_config: "mypy.ini" enable_threshold_check: "true" enable_base_tip_check: "true" github_token: ${{ secrets.GITHUB_TOKEN }} From a1ef6c0e9a13eac295dfa7350ce060df920c3ad1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Tue, 21 Nov 2023 16:11:25 +0100 Subject: [PATCH 120/158] Revert "Add mypy check" --- .github/workflows/lint-code.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 6756597adf..edff9a8c82 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -6,7 +6,6 @@ on: pull_request: release: types: [published] - workflow_dispatch: # Cancel if a newer run is started concurrency: @@ -126,13 +125,3 @@ jobs: - name: Run if any of the listed files above is changed if: steps.changed-py-files.outputs.any_changed == 'true' run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} - - - name: MyPy Check - uses: photonbit/mypy-imprecision-action@main - with: - max_tip: "60" - sufficient_tip: "10" - enable_threshold_check: "true" - enable_base_tip_check: "true" - github_token: ${{ secrets.GITHUB_TOKEN }} - if: ${{ github.event_name == 'pull_request' && github.head_ref == 'dev' }} From 41d8fe2836fbf7c3570f191c9d37c82a74ef6e75 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Tue, 21 Nov 2023 18:42:53 +0100 Subject: [PATCH 121/158] linting tests requiring nf-test will be warnings instead of failures --- nf_core/modules/lint/module_tests.py | 8 ++++---- nf_core/subworkflows/lint/subworkflow_tests.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 3dffefeaa9..d27ae15841 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -20,14 +20,14 @@ def module_tests(_, module): if module.nftest_testdir.is_dir(): module.passed.append(("test_dir_exists", "nf-test test directory exists", module.nftest_testdir)) else: - module.failed.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) + module.warned.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) return # Lint the test main.nf file if module.nftest_main_nf.is_file(): module.passed.append(("test_main_exists", "test `main.nf.test` exists", module.nftest_main_nf)) else: - module.failed.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) + module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) if module.nftest_main_nf.is_file(): # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test @@ -104,7 +104,7 @@ def module_tests(_, module): with open(pytest_yml_path, "r") as fh: pytest_yml = yaml.safe_load(fh) if module.component_name in pytest_yml.keys(): - module.failed.append( + module.warned.append( ( "test_pytest_yml", "module with nf-test should not be listed in pytest_modules.yml", @@ -138,4 +138,4 @@ def module_tests(_, module): ) ) else: - module.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) + module.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index f457841c99..8d07cefd42 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -24,14 +24,14 @@ def subworkflow_tests(_, subworkflow): if subworkflow.nftest_testdir.is_dir(): subworkflow.passed.append(("test_dir_exists", "nf-test test directory exists", subworkflow.nftest_testdir)) else: - subworkflow.failed.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) + subworkflow.warned.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) return # Lint the test main.nf file if subworkflow.nftest_main_nf.is_file(): subworkflow.passed.append(("test_main_exists", "test `main.nf.test` exists", subworkflow.nftest_main_nf)) else: - subworkflow.failed.append( + subworkflow.warned.append( ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) ) @@ -114,7 +114,7 @@ def subworkflow_tests(_, subworkflow): with open(pytest_yml_path, "r") as fh: pytest_yml = yaml.safe_load(fh) if "subworkflows/" + subworkflow.component_name in pytest_yml.keys(): - subworkflow.failed.append( + subworkflow.warned.append( ( "test_pytest_yml", "subworkflow with nf-test should not be listed in pytest_modules.yml", @@ -151,4 +151,4 @@ def subworkflow_tests(_, subworkflow): ) ) else: - subworkflow.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) + subworkflow.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) From 7fe54c693b699dd99342a45c79df11f310c55cce Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 22:58:57 +0100 Subject: [PATCH 122/158] keep linting failing for component with nf-test --- nf_core/modules/lint/module_tests.py | 29 ++++++++++++---- .../subworkflows/lint/subworkflow_tests.py | 33 ++++++++++++++----- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index d27ae15841..1a96dc7227 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -2,14 +2,16 @@ Lint the tests of a module in nf-core/modules """ import logging -import os +from pathlib import Path import yaml +from nf_core.components.nfcore_component import NFCoreComponent + log = logging.getLogger(__name__) -def module_tests(_, module): +def module_tests(_, module: NFCoreComponent): """ Lint the tests of a module in ``nf-core/modules`` @@ -17,17 +19,27 @@ def module_tests(_, module): and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. """ + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] + test_dir = Path(module.base_dir, "tests", "module", repo_dir, module.component_name) + pytest_main_nf = Path(test_dir, "main.nf") + is_pytest = pytest_main_nf.is_file() if module.nftest_testdir.is_dir(): module.passed.append(("test_dir_exists", "nf-test test directory exists", module.nftest_testdir)) else: - module.warned.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) + if is_pytest: + module.warned.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) + else: + module.failed.append(("test_dir_exists", "nf-test directory is missing", module.nftest_testdir)) return # Lint the test main.nf file if module.nftest_main_nf.is_file(): module.passed.append(("test_main_exists", "test `main.nf.test` exists", module.nftest_main_nf)) else: - module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) + if is_pytest: + module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) + else: + module.failed.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) if module.nftest_main_nf.is_file(): # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test @@ -99,12 +111,12 @@ def module_tests(_, module): # Check pytest_modules.yml does not contain entries for modules with nf-test pytest_yml_path = module.base_dir / "tests" / "config" / "pytest_modules.yml" - if pytest_yml_path.is_file(): + if pytest_yml_path.is_file() and not is_pytest: try: with open(pytest_yml_path, "r") as fh: pytest_yml = yaml.safe_load(fh) if module.component_name in pytest_yml.keys(): - module.warned.append( + module.failed.append( ( "test_pytest_yml", "module with nf-test should not be listed in pytest_modules.yml", @@ -138,4 +150,7 @@ def module_tests(_, module): ) ) else: - module.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) + if is_pytest: + module.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) + else: + module.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8d07cefd42..41ebe932e8 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -7,8 +7,6 @@ import yaml -import nf_core.subworkflows - log = logging.getLogger(__name__) @@ -21,19 +19,33 @@ def subworkflow_tests(_, subworkflow): Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` """ + + repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] + test_dir = Path(module.base_dir, "tests", "module", repo_dir, module.component_name) + pytest_main_nf = Path(test_dir, "main.nf") + is_pytest = pytest_main_nf.is_file() + if subworkflow.nftest_testdir.is_dir(): subworkflow.passed.append(("test_dir_exists", "nf-test test directory exists", subworkflow.nftest_testdir)) else: - subworkflow.warned.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) + if is_pytest: + subworkflow.warned.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) + else: + subworkflow.failed.append(("test_dir_exists", "nf-test directory is missing", subworkflow.nftest_testdir)) return # Lint the test main.nf file if subworkflow.nftest_main_nf.is_file(): subworkflow.passed.append(("test_main_exists", "test `main.nf.test` exists", subworkflow.nftest_main_nf)) else: - subworkflow.warned.append( - ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) - ) + if is_pytest: + subworkflow.warned.append( + ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) + ) + else: + subworkflow.failed.append( + ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) + ) if subworkflow.nftest_main_nf.is_file(): with open(subworkflow.nftest_main_nf, "r") as fh: @@ -109,12 +121,12 @@ def subworkflow_tests(_, subworkflow): # Check pytest_modules.yml does not contain entries for subworkflows with nf-test pytest_yml_path = subworkflow.base_dir / "tests" / "config" / "pytest_modules.yml" - if pytest_yml_path.is_file(): + if pytest_yml_path.is_file() and not is_pytest: try: with open(pytest_yml_path, "r") as fh: pytest_yml = yaml.safe_load(fh) if "subworkflows/" + subworkflow.component_name in pytest_yml.keys(): - subworkflow.warned.append( + subworkflow.failed.append( ( "test_pytest_yml", "subworkflow with nf-test should not be listed in pytest_modules.yml", @@ -151,4 +163,7 @@ def subworkflow_tests(_, subworkflow): ) ) else: - subworkflow.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) + if is_pytest: + subworkflow.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) + else: + subworkflow.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) From 3446c0bdcb7b24584e89ee609c388b596b685cd2 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 21 Nov 2023 23:01:11 +0100 Subject: [PATCH 123/158] fix typo --- nf_core/modules/lint/module_tests.py | 2 +- nf_core/subworkflows/lint/subworkflow_tests.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 1a96dc7227..8188787f90 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -20,7 +20,7 @@ def module_tests(_, module: NFCoreComponent): """ repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] - test_dir = Path(module.base_dir, "tests", "module", repo_dir, module.component_name) + test_dir = Path(module.base_dir, "tests", "modules", repo_dir, module.component_name) pytest_main_nf = Path(test_dir, "main.nf") is_pytest = pytest_main_nf.is_file() if module.nftest_testdir.is_dir(): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 41ebe932e8..8e76c03ceb 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -2,15 +2,16 @@ Lint the tests of a subworkflow in nf-core/modules """ import logging -import os from pathlib import Path import yaml +from nf_core.components.nfcore_component import NFCoreComponent + log = logging.getLogger(__name__) -def subworkflow_tests(_, subworkflow): +def subworkflow_tests(_, subworkflow: NFCoreComponent): """ Lint the tests of a subworkflow in ``nf-core/modules`` @@ -20,8 +21,10 @@ def subworkflow_tests(_, subworkflow): Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` """ - repo_dir = module.component_dir.parts[: module.component_dir.parts.index(module.component_name.split("/")[0])][-1] - test_dir = Path(module.base_dir, "tests", "module", repo_dir, module.component_name) + repo_dir = subworkflow.component_dir.parts[ + : subworkflow.component_dir.parts.index(subworkflow.component_name.split("/")[0]) + ][-1] + test_dir = Path(subworkflow.base_dir, "tests", "subworfklows", repo_dir, subworkflow.component_name) pytest_main_nf = Path(test_dir, "main.nf") is_pytest = pytest_main_nf.is_file() From e6d6f2cecca4311d1244d7f5be343e0e3b049b68 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 23 Nov 2023 13:08:04 +0100 Subject: [PATCH 124/158] move set_wd from tests/utils.py to utils.py --- nf_core/components/components_test.py | 3 +-- nf_core/utils.py | 21 ++++++++++++++++++++- tests/components/generate_snapshot.py | 3 ++- tests/components/snapshot_test.py | 3 +-- tests/modules/lint.py | 3 ++- tests/test_test_utils.py | 19 +------------------ tests/test_utils.py | 17 +++++++++++++++++ tests/utils.py | 18 ------------------ 8 files changed, 44 insertions(+), 43 deletions(-) diff --git a/nf_core/components/components_test.py b/nf_core/components/components_test.py index 689a6819eb..06cc11f3ba 100644 --- a/nf_core/components/components_test.py +++ b/nf_core/components/components_test.py @@ -19,7 +19,6 @@ import nf_core.utils from nf_core.components.components_command import ComponentCommand -from tests.utils import set_wd log = logging.getLogger(__name__) @@ -92,7 +91,7 @@ def run(self) -> None: os.environ[ "NFT_DIFF_ARGS" ] = "--line-numbers --expand-tabs=2" # taken from https://code.askimed.com/nf-test/docs/assertions/snapshots/#snapshot-differences - with set_wd(Path(self.dir)): + with nf_core.utils.set_wd(Path(self.dir)): self.check_snapshot_stability() if len(self.errors) > 0: errors = "\n - ".join(self.errors) diff --git a/nf_core/utils.py b/nf_core/utils.py index c57990a9d3..eda0ed8f55 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -16,8 +16,9 @@ import subprocess import sys import time +from contextlib import contextmanager from pathlib import Path -from typing import Tuple, Union +from typing import Generator, Tuple, Union import git import prompt_toolkit @@ -1147,3 +1148,21 @@ def validate_file_md5(file_name, expected_md5hex): raise IOError(f"{file_name} md5 does not match remote: {expected_md5hex} - {file_md5hex}") return True + + +@contextmanager +def set_wd(path: Path) -> Generator[None, None, None]: + """Sets the working directory for this context. + + Arguments + --------- + + path : Path + Path to the working directory to be used inside this context. + """ + start_wd = Path().absolute() + os.chdir(Path(path).resolve()) + try: + yield + finally: + os.chdir(start_wd) diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index c7eb696722..46fd63fe3f 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -6,8 +6,9 @@ import pytest from nf_core.components.components_test import ComponentsTest +from nf_core.utils import set_wd -from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL, set_wd +from ..utils import GITLAB_NFTEST_BRANCH, GITLAB_URL def test_generate_snapshot_module(self): diff --git a/tests/components/snapshot_test.py b/tests/components/snapshot_test.py index 371f0d6fbe..d774618476 100644 --- a/tests/components/snapshot_test.py +++ b/tests/components/snapshot_test.py @@ -5,8 +5,7 @@ import pytest from nf_core.components.components_test import ComponentsTest - -from ..utils import set_wd +from nf_core.utils import set_wd def test_components_test_check_inputs(self): diff --git a/tests/modules/lint.py b/tests/modules/lint.py index a3a7a80b71..f35b7eee1e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -5,8 +5,9 @@ import nf_core.modules from nf_core.modules.lint import main_nf +from nf_core.utils import set_wd -from ..utils import GITLAB_URL, set_wd +from ..utils import GITLAB_URL from .patch import BISMARK_ALIGN, CORRECT_SHA, PATCH_BRANCH, REPO_NAME, modify_main_nf diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index c4e3d49ae0..154a31fca6 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -3,7 +3,7 @@ import pytest -from .utils import set_wd, with_temporary_file, with_temporary_folder +from .utils import with_temporary_file, with_temporary_folder def test_with_temporary_file(): @@ -30,20 +30,3 @@ def tmp_folder_exists(tmp_folder): def test_tmp_folder_does_not_exist_after(): tmp_folder = with_temporary_folder(lambda x: x)() assert not Path(tmp_folder).exists() - - -def test_set_wd(): - with tempfile.TemporaryDirectory() as tmpdirname: - with set_wd(tmpdirname): - context_wd = Path().resolve() - assert context_wd == Path(tmpdirname).resolve() - assert context_wd != Path().resolve() - - -def test_set_wd_revert_on_raise(): - wd_before_context = Path().resolve() - with tempfile.TemporaryDirectory() as tmpdirname: - with pytest.raises(Exception): - with set_wd(tmpdirname): - raise Exception - assert wd_before_context == Path().resolve() diff --git a/tests/test_utils.py b/tests/test_utils.py index 2ab5b64bfc..56e83ef190 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -207,3 +207,20 @@ def test_validate_file_md5(): nf_core.utils.validate_file_md5(test_file, different_md5) with pytest.raises(ValueError): nf_core.utils.validate_file_md5(test_file, non_hex_string) + + +def test_set_wd(): + with tempfile.TemporaryDirectory() as tmpdirname: + with nf_core.utils.set_wd(tmpdirname): + context_wd = Path().resolve() + assert context_wd == Path(tmpdirname).resolve() + assert context_wd != Path().resolve() + + +def test_set_wd_revert_on_raise(): + wd_before_context = Path().resolve() + with tempfile.TemporaryDirectory() as tmpdirname: + with pytest.raises(Exception): + with nf_core.utils.set_wd(tmpdirname): + raise Exception + assert wd_before_context == Path().resolve() diff --git a/tests/utils.py b/tests/utils.py index 307129b5b2..198ac3d583 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -59,24 +59,6 @@ def wrapper(*args: Any, **kwargs: Any) -> Any: return wrapper -@contextmanager -def set_wd(path: Path) -> Generator[None, None, None]: - """Sets the working directory for this context. - - Arguments - --------- - - path : Path - Path to the working directory to be used iside this context. - """ - start_wd = Path().absolute() - os.chdir(Path(path).resolve()) - try: - yield - finally: - os.chdir(start_wd) - - def mock_anaconda_api_calls(rsps: responses.RequestsMock, module: str, version: str) -> None: """Mock anaconda api calls for module""" anaconda_api_url = f"https://api.anaconda.org/package/bioconda/{module}" From e3113d9123cc9ec59b549ca0dd35f2748a602eb9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 23 Nov 2023 17:30:07 +0100 Subject: [PATCH 125/158] lint for removal of old pytest files --- nf_core/modules/lint/module_tests.py | 14 ++++++++++++++ nf_core/subworkflows/lint/subworkflow_tests.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index 8188787f90..e172d35a3e 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -154,3 +154,17 @@ def module_tests(_, module: NFCoreComponent): module.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) else: module.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", module.tags_yml)) + + # Check that the old test directory does not exist + if not is_pytest: + old_test_dir = Path(module.base_dir, "tests", "modules", module.component_name) + if old_test_dir.is_dir(): + module.failed.append( + ( + "test_old_test_dir", + f"Pytest files are still present at `{Path('tests', 'modules', module.component_name)}`. Please remove this directory and its contents.", + old_test_dir, + ) + ) + else: + module.passed.append(("test_old_test_dir", "Old pytests don't exist for this module", old_test_dir)) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 8e76c03ceb..81ad06c2bf 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -170,3 +170,11 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): subworkflow.warned.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) else: subworkflow.failed.append(("test_tags_yml_exists", "file `tags.yml` does not exist", subworkflow.tags_yml)) + + # Check that the old test directory does not exist + if not is_pytest: + old_test_dir = Path(subworkflow.base_dir, "tests", "subworkflows", subworkflow.component_name) + if old_test_dir.is_dir(): + subworkflow.failed.append(("test_old_test_dir", "old test directory exists", old_test_dir)) + else: + subworkflow.passed.append(("test_old_test_dir", "old test directory does not exist", old_test_dir)) From fbc228e382d973746b9e3b6075de35a1c77e0f4b Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 23 Nov 2023 17:30:44 +0100 Subject: [PATCH 126/158] tests, tests, tests --- nf_core/modules/lint/module_tests.py | 18 ++- .../subworkflows/lint/subworkflow_tests.py | 6 +- tests/modules/lint.py | 111 ++++++++++++++++++ tests/test_modules.py | 8 +- 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index e172d35a3e..1a18482972 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -34,12 +34,12 @@ def module_tests(_, module: NFCoreComponent): # Lint the test main.nf file if module.nftest_main_nf.is_file(): - module.passed.append(("test_main_exists", "test `main.nf.test` exists", module.nftest_main_nf)) + module.passed.append(("test_main_nf_exists", "test `main.nf.test` exists", module.nftest_main_nf)) else: if is_pytest: - module.warned.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) + module.warned.append(("test_main_nf_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) else: - module.failed.append(("test_main_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) + module.failed.append(("test_main_nf_exists", "test `main.nf.test` does not exist", module.nftest_main_nf)) if module.nftest_main_nf.is_file(): # Check if main.nf.test.snap file exists, if 'snap(' is inside main.nf.test @@ -104,7 +104,7 @@ def module_tests(_, module: NFCoreComponent): module.failed.append( ( "test_main_tags", - f"Tags do not adhere to guidelines. Tags missing in `main.nf.test`: {missing_tags}", + f"Tags do not adhere to guidelines. Tags missing in `main.nf.test`: `{','.join(missing_tags)}`", module.nftest_main_nf, ) ) @@ -140,12 +140,18 @@ def module_tests(_, module: NFCoreComponent): if f"modules/{module.org}/{module.component_name}/**" in tags_yml[module.component_name]: module.passed.append(("test_tags_yml", "correct path in tags.yml", module.tags_yml)) else: - module.failed.append(("test_tags_yml", "incorrect path in tags.yml", module.tags_yml)) + module.failed.append( + ( + "test_tags_yml", + f"incorrect path in tags.yml, expected `modules/{module.org}/{module.component_name}/**`, got `{tags_yml[module.component_name][0]}`", + module.tags_yml, + ) + ) else: module.failed.append( ( "test_tags_yml", - "incorrect entry in tags.yml, should be '' or '/'", + f"incorrect key in tags.yml, should be `{module.component_name}`, got `{list(tags_yml.keys())[0]}`.", module.tags_yml, ) ) diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 81ad06c2bf..e82d00f636 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -39,15 +39,15 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): # Lint the test main.nf file if subworkflow.nftest_main_nf.is_file(): - subworkflow.passed.append(("test_main_exists", "test `main.nf.test` exists", subworkflow.nftest_main_nf)) + subworkflow.passed.append(("test_main_nf_exists", "test `main.nf.test` exists", subworkflow.nftest_main_nf)) else: if is_pytest: subworkflow.warned.append( - ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) + ("test_main_nf_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) ) else: subworkflow.failed.append( - ("test_main_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) + ("test_main_nf_exists", "test `main.nf.test` does not exist", subworkflow.nftest_main_nf) ) if subworkflow.nftest_main_nf.is_file(): diff --git a/tests/modules/lint.py b/tests/modules/lint.py index f35b7eee1e..801c906a69 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -420,3 +420,114 @@ def test_modules_environment_yml_file_name_mismatch(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "environment_yml_name" + + +def test_modules_missing_test_dir(self): + """Test linting a module with a missing test directory""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests.bak") + ) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests.bak").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests") + ) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_dir_exists" + + +def test_modules_missing_test_main_nf(self): + """Test linting a module with a missing test/main.nf file""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.bak") + ) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.bak").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test") + ) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_main_nf_exists" + + +def test_modules_missing_required_tag(self): + """Test linting a module with a missing required tag""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "r") as fh: + content = fh.read() + new_content = content.replace("modules_nfcore", "foo") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "w") as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test"), "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_main_tags" + + +def test_modules_missing_tags_yml(self): + """Test linting a module with a missing tags.yml file""" + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak") + ) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak").rename( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + ) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml_exists" + + +def test_modules_incorrect_tags_yml_key(self): + """Test linting a module with an incorrect key in tags.yml file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "r") as fh: + content = fh.read() + new_content = content.replace("bpipe/test:", "bpipe_test:") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=True, module="bpipe/test") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + + +def test_modules_incorrect_tags_yml_values(self): + """Test linting a module with an incorrect path in tags.yml file""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "r") as fh: + content = fh.read() + new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + + +def test_modules_unused_pytest_files(self): + """Test linting a nf-test module with files still present in `tests/modules/`""" + Path(self.nfcore_modules, "tests", "modules", "bpipe", "test").mkdir(parents=True, exist_ok=True) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + Path(self.nfcore_modules, "tests", "modules", "bpipe", "test").rmdir() + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_old_test_dir" diff --git a/tests/test_modules.py b/tests/test_modules.py index 39c600b986..59f0b368a0 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -3,7 +3,6 @@ import os import shutil -import tempfile import unittest from pathlib import Path @@ -164,6 +163,8 @@ def test_modulesrepo_class(self): test_modules_environment_yml_file_not_array, test_modules_environment_yml_file_sorted_correctly, test_modules_environment_yml_file_sorted_incorrectly, + test_modules_incorrect_tags_yml_key, + test_modules_incorrect_tags_yml_values, test_modules_lint_check_process_labels, test_modules_lint_check_url, test_modules_lint_empty, @@ -176,6 +177,11 @@ def test_modulesrepo_class(self): test_modules_lint_snapshot_file_missing_fail, test_modules_lint_snapshot_file_not_needed, test_modules_lint_trimgalore, + test_modules_missing_required_tag, + test_modules_missing_tags_yml, + test_modules_missing_test_dir, + test_modules_missing_test_main_nf, + test_modules_unused_pytest_files, ) from .modules.list import ( # type: ignore[misc] test_modules_install_and_list_pipeline, From 678d22e4888f0eba412ce2f0aca3828e7723b6d5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Thu, 23 Nov 2023 17:42:44 +0100 Subject: [PATCH 127/158] nudge people to snapshot all of their output --- nf_core/module-template/modules/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/modules/tests/main.nf.test index 883a3ffa40..105c70eb96 100644 --- a/nf_core/module-template/modules/tests/main.nf.test +++ b/nf_core/module-template/modules/tests/main.nf.test @@ -40,7 +40,7 @@ nextflow_process { then { assertAll( { assert process.success }, - { assert snapshot(process.out.versions).match("versions") } + { assert snapshot(process.out).match() } //TODO nf-core: Add all required assertions to verify the test output. ) } From 4157d785fcae445717f588020ff631533bdfe4b9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 10:28:34 +0100 Subject: [PATCH 128/158] improve testing coverage for meta_yml --- nf_core/components/nfcore_component.py | 60 +++++++++++++++-- nf_core/modules/bump_versions.py | 2 +- nf_core/modules/lint/meta_yml.py | 90 ++++++++++++++++++++++--- nf_core/modules/modules_utils.py | 10 +-- tests/modules/lint.py | 91 ++++++++++++++++++++++++++ tests/test_modules.py | 37 ++++++++--- 6 files changed, 258 insertions(+), 32 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 94e7584a9b..c26db1cf99 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -1,8 +1,12 @@ """ The NFCoreComponent class holds information and utility functions for a single module or subworkflow """ +import logging +import re from pathlib import Path +log = logging.getLogger(__name__) + class NFCoreComponent: """ @@ -44,16 +48,16 @@ def __init__( if remote_component: # Initialize the important files - self.main_nf = self.component_dir / "main.nf" - self.meta_yml = self.component_dir / "meta.yml" + self.main_nf = Path(self.component_dir, "main.nf") + self.meta_yml = Path(self.component_dir, "meta.yml") self.process_name = "" - self.environment_yml = self.component_dir / "environment.yml" + self.environment_yml = Path(self.component_dir, "environment.yml") repo_dir = self.component_dir.parts[: self.component_dir.parts.index(self.component_name.split("/")[0])][-1] self.org = repo_dir - self.nftest_testdir = self.component_dir / "tests" - self.nftest_main_nf = self.nftest_testdir / "main.nf.test" - self.tags_yml = self.nftest_testdir / "tags.yml" + self.nftest_testdir = Path(self.component_dir, "tests") + self.nftest_main_nf = Path(self.nftest_testdir, "main.nf.test") + self.tags_yml = Path(self.nftest_testdir, "tags.yml") if self.repo_type == "pipeline": patch_fn = f"{self.component_name.replace('/', '-')}.diff" @@ -90,3 +94,47 @@ def _get_included_components(self, main_nf: str): if line.strip().startswith("include"): included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) return included_components + + def get_inputs_from_main_nf(self): + """Collect all inputs from the main.nf file.""" + inputs = [] + with open(self.main_nf, "r") as f: + data = f.read() + # get input values from main.nf after "input:", which can be formatted as tuple val(foo) path(bar) or val foo or val bar or path bar or path foo + # regex matches: + # val(foo) + # path(bar) + # val foo + # val bar + # path bar + # path foo + # don't match anything inside comments or after "output:" + if "input:" not in data: + log.debug(f"Could not find any inputs in {self.main_nf}") + return inputs + input_data = data.split("input:")[1].split("output:")[0] + regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s]+))" + matches = re.finditer(regex, input_data, re.MULTILINE) + for matchNum, match in enumerate(matches, start=1): + if match.group(3): + inputs.append(match.group(3)) + elif match.group(4): + inputs.append(match.group(4)) + log.info(f"Found {len(inputs)} inputs in {self.main_nf}") + self.inputs = inputs + + def get_outputs_from_main_nf(self): + outputs = [] + with open(self.main_nf, "r") as f: + data = f.read() + # get output values from main.nf after "output:". the names are always after "emit:" + if "output:" not in data: + log.info(f"Could not find any outputs in {self.main_nf}") + return outputs + output_data = data.split("output:")[1].split("when:")[0] + regex = r"emit:\s*([^)\s]+)" + matches = re.finditer(regex, output_data, re.MULTILINE) + for matchNum, match in enumerate(matches, start=1): + outputs.append(match.group(1)) + log.info(f"Found {len(outputs)} outputs in {self.main_nf}") + self.outputs = outputs diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index cb77d4c043..25259f1a16 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -49,7 +49,7 @@ def __init__( self.tools_config: Dict[str, Any] = {} def bump_versions( - self, module: Union[NFCoreComponent, None] = None, all_modules: bool = False, show_uptodate: bool = False + self, module: Union[str, None] = None, all_modules: bool = False, show_uptodate: bool = False ) -> None: """ Bump the container and conda version of single module or all modules diff --git a/nf_core/modules/lint/meta_yml.py b/nf_core/modules/lint/meta_yml.py index 6bdc4ed223..440b54a310 100644 --- a/nf_core/modules/lint/meta_yml.py +++ b/nf_core/modules/lint/meta_yml.py @@ -22,7 +22,24 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None and module input is consistent between the ``meta.yml`` and the ``main.nf``. + If the module has inputs or outputs, they are expected to be + formatted as: + + ..code-block:: + tuple val(foo) path(bar) + val foo + path foo + + or permutations of the above. + + Args: + module_lint_object (ComponentLint): The lint object for the module + module (NFCoreComponent): The module to lint + """ + + module.get_inputs_from_main_nf() + module.get_outputs_from_main_nf() # Check if we have a patch file, get original file in that case meta_yaml = None if module.is_patched: @@ -45,14 +62,14 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None return # Confirm that the meta.yml file is valid according to the JSON schema - valid_meta_yml = True + valid_meta_yml = False try: with open(Path(module_lint_object.modules_repo.local_repo_dir, "modules/meta-schema.json"), "r") as fh: schema = json.load(fh) validators.validate(instance=meta_yaml, schema=schema) module.passed.append(("meta_yml_valid", "Module `meta.yml` is valid", module.meta_yml)) + valid_meta_yml = True except exceptions.ValidationError as e: - valid_meta_yml = False hint = "" if len(e.path) > 0: hint = f"\nCheck the entry for `{e.path[0]}`." @@ -79,26 +96,79 @@ def meta_yml(module_lint_object: ComponentLint, module: NFCoreComponent) -> None meta_input = [list(x.keys())[0] for x in meta_yaml["input"]] for input in module.inputs: if input in meta_input: - module.passed.append(("meta_input", f"`{input}` specified", module.meta_yml)) + module.passed.append(("meta_input_main_only", f"`{input}` specified", module.meta_yml)) + else: + module.warned.append( + ( + "meta_input_main_only", + f"`{input}` is present as an input in the `main.nf`, but missing in `meta.yml`", + module.meta_yml, + ) + ) + # check if there are any inputs in meta.yml that are not in main.nf + for input in meta_input: + if input in module.inputs: + module.passed.append( + ( + "meta_input_meta_only", + f"`{input}` is present as an input in `meta.yml` and `main.nf`", + module.meta_yml, + ) + ) else: - module.failed.append(("meta_input", f"`{input}` missing in `meta.yml`", module.meta_yml)) + module.warned.append( + ( + "meta_input_meta_only", + f"`{input}` is present as an input in `meta.yml` but not in `main.nf`", + module.meta_yml, + ) + ) - if "output" in meta_yaml: + if "output" in meta_yaml and meta_yaml["output"] is not None: meta_output = [list(x.keys())[0] for x in meta_yaml["output"]] for output in module.outputs: if output in meta_output: - module.passed.append(("meta_output", f"`{output}` specified", module.meta_yml)) + module.passed.append(("meta_output_main_only", f"`{output}` specified", module.meta_yml)) else: - module.failed.append(("meta_output", f"`{output}` missing in `meta.yml`", module.meta_yml)) - + module.warned.append( + ( + "meta_output_main_only", + f"`{output}` is present as an output in the `main.nf`, but missing in `meta.yml`", + module.meta_yml, + ) + ) + # check if there are any outputs in meta.yml that are not in main.nf + for output in meta_output: + if output in module.outputs: + module.passed.append( + ( + "meta_output_meta_only", + f"`{output}` is present as an output in `meta.yml` and `main.nf`", + module.meta_yml, + ) + ) + else: + module.warned.append( + ( + "meta_output_meta_only", + f"`{output}` is present as an output in `meta.yml` but not in `main.nf`", + module.meta_yml, + ) + ) # confirm that the name matches the process name in main.nf if meta_yaml["name"].upper() == module.process_name: - module.passed.append(("meta_name", "Correct name specified in `meta.yml`", module.meta_yml)) + module.passed.append( + ( + "meta_name", + "Correct name specified in `meta.yml`.", + module.meta_yml, + ) + ) else: module.failed.append( ( "meta_name", - f"Conflicting process name between meta.yml (`{meta_yaml['name']}`) and main.nf (`{module.process_name}`)", + f"Conflicting `process` name between meta.yml (`{meta_yaml['name']}`) and main.nf (`{module.process_name}`)", module.meta_yml, ) ) diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index 504cb1095d..3ae01e9eef 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -37,7 +37,7 @@ def repo_full_name_from_remote(remote_url: str) -> str: return path -def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[str]]: +def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], List[NFCoreComponent]]: """ Make a list of all modules installed in this repository @@ -52,7 +52,7 @@ def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], Lis """ # initialize lists local_modules: List[str] = [] - nfcore_modules: List[str] = [] + nfcore_modules_names: List[str] = [] local_modules_dir: Optional[str] = None nfcore_modules_dir = os.path.join(dir, "modules", "nf-core") @@ -76,9 +76,9 @@ def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], Lis # Not a module, but contains sub-modules if not "main.nf" in m_content: for tool in m_content: - nfcore_modules.append(os.path.join(m, tool)) + nfcore_modules_names.append(os.path.join(m, tool)) else: - nfcore_modules.append(m) + nfcore_modules_names.append(m) # Make full (relative) file paths and create NFCoreComponent objects if local_modules_dir: @@ -93,7 +93,7 @@ def get_installed_modules(dir: str, repo_type="modules") -> Tuple[List[str], Lis base_dir=Path(dir), component_type="modules", ) - for m in nfcore_modules + for m in nfcore_modules_names ] return local_modules, nfcore_modules diff --git a/tests/modules/lint.py b/tests/modules/lint.py index f35b7eee1e..1c953989a6 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -420,3 +420,94 @@ def test_modules_environment_yml_file_name_mismatch(self): assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "environment_yml_name" + + +def test_modules_meta_yml_incorrect_licence_field(self): + """Test linting a module with an incorrect Licence field in meta.yml""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml")) as fh: + meta_yml = yaml.safe_load(fh) + meta_yml["tools"][0]["bpipe"]["licence"] = "[MIT]" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml"), "w") as fh: + fh.write(yaml.dump(meta_yml)) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + + # reset changes + meta_yml["tools"][0]["bpipe"]["licence"] = ["MIT"] + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml"), "w") as fh: + fh.write(yaml.dump(meta_yml)) + + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "meta_yml_valid" + + +def test_modules_meta_yml_input_mismatch(self): + """Test linting a module with an extra entry in input fields in meta.yml compared to module.input""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf")) as fh: + main_nf = fh.read() + main_nf_new = main_nf.replace("path bam", "path bai") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: + fh.write(main_nf_new) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: + fh.write(main_nf) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) == 2 + lint_tests = [x.lint_test for x in module_lint.warned] + # check that it is there twice: + assert lint_tests.count("meta_input_meta_only") == 1 + assert lint_tests.count("meta_input_main_only") == 1 + + +def test_modules_meta_yml_output_mismatch(self): + """Test linting a module with an extra entry in output fields in meta.yml compared to module.output""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf")) as fh: + main_nf = fh.read() + main_nf_new = main_nf.replace("emit: bam", "emit: bai") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: + fh.write(main_nf_new) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "main.nf"), "w") as fh: + fh.write(main_nf) + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) == 2 + lint_tests = [x.lint_test for x in module_lint.warned] + # check that it is there twice: + assert lint_tests.count("meta_output_meta_only") == 1 + assert lint_tests.count("meta_output_main_only") == 1 + + +def test_modules_meta_yml_incorrect_name(self): + """Test linting a module with an incorrect name in meta.yml""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml")) as fh: + meta_yml = yaml.safe_load(fh) + meta_yml["name"] = "bpipe/test" + # need to make the same change to the environment.yml file + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml")) as fh: + environment_yml = yaml.safe_load(fh) + environment_yml["name"] = "bpipe/test" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml"), "w") as fh: + fh.write(yaml.dump(meta_yml)) + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml.dump(environment_yml)) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + + # reset changes + meta_yml["name"] = "bpipe_test" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "meta.yml"), "w") as fh: + fh.write(yaml.dump(meta_yml)) + environment_yml["name"] = "bpipe_test" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "environment.yml"), "w") as fh: + fh.write(yaml.dump(environment_yml)) + + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "meta_name" diff --git a/tests/test_modules.py b/tests/test_modules.py index 39c600b986..7aeb725da1 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -3,12 +3,12 @@ import os import shutil -import tempfile import unittest from pathlib import Path import requests_cache import responses +import yaml import nf_core.create import nf_core.modules @@ -45,16 +45,29 @@ def create_modules_repo_dummy(tmp_dir): module_create.create() # Remove doi from meta.yml which makes lint fail - meta_yml = Path(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") + meta_yml_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "meta.yml") Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap").touch() - with open(meta_yml, "r") as fh: - lines = fh.readlines() - for line_index in range(len(lines)): - if "doi" in lines[line_index]: - to_pop = line_index - lines.pop(to_pop) - with open(meta_yml, "w") as fh: - fh.writelines(lines) + with open(meta_yml_path, "r") as fh: + meta_yml = yaml.safe_load(fh) + del meta_yml["tools"][0]["bpipe"]["doi"] + with open(meta_yml_path, "w") as fh: + yaml.dump(meta_yml, fh) + + # remove "TODO" statements from main.nf + main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") + with open(main_nf_path, "r") as fh: + main_nf = fh.read() + main_nf = main_nf.replace("TODO", "") + with open(main_nf_path, "w") as fh: + fh.write(main_nf) + + # remove "TODO" statements from main.nf.test + main_nf_test_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test") + with open(main_nf_test_path, "r") as fh: + main_nf_test = fh.read() + main_nf_test = main_nf_test.replace("TODO", "") + with open(main_nf_test_path, "w") as fh: + fh.write(main_nf_test) return root_dir @@ -176,6 +189,10 @@ def test_modulesrepo_class(self): test_modules_lint_snapshot_file_missing_fail, test_modules_lint_snapshot_file_not_needed, test_modules_lint_trimgalore, + test_modules_meta_yml_incorrect_licence_field, + test_modules_meta_yml_incorrect_name, + test_modules_meta_yml_input_mismatch, + test_modules_meta_yml_output_mismatch, ) from .modules.list import ( # type: ignore[misc] test_modules_install_and_list_pipeline, From 955443b3624d2ccc17e1d3cedb9613f8baf08ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 27 Nov 2023 11:32:55 +0100 Subject: [PATCH 129/158] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Júlia Mir Pedrol --- nf_core/components/nfcore_component.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index c26db1cf99..6a002117f0 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -113,7 +113,7 @@ def get_inputs_from_main_nf(self): log.debug(f"Could not find any inputs in {self.main_nf}") return inputs input_data = data.split("input:")[1].split("output:")[0] - regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s]+))" + regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" matches = re.finditer(regex, input_data, re.MULTILINE) for matchNum, match in enumerate(matches, start=1): if match.group(3): @@ -132,7 +132,7 @@ def get_outputs_from_main_nf(self): log.info(f"Could not find any outputs in {self.main_nf}") return outputs output_data = data.split("output:")[1].split("when:")[0] - regex = r"emit:\s*([^)\s]+)" + regex = r"emit:\s*([^)\s,]+)" matches = re.finditer(regex, output_data, re.MULTILINE) for matchNum, match in enumerate(matches, start=1): outputs.append(match.group(1)) From 8ef06217998d69d8e8c6520d896caa47b48b2e07 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 11:50:38 +0100 Subject: [PATCH 130/158] log to info instead of debug. --- nf_core/components/nfcore_component.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 6a002117f0..190310faae 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -110,7 +110,7 @@ def get_inputs_from_main_nf(self): # path foo # don't match anything inside comments or after "output:" if "input:" not in data: - log.debug(f"Could not find any inputs in {self.main_nf}") + log.info(f"Could not find any inputs in {self.main_nf}") return inputs input_data = data.split("input:")[1].split("output:")[0] regex = r"(val|path)\s*(\(([^)]+)\)|\s*([^)\s,]+))" From 9d4bd0e568ddf7345aae3cadf2c02fa175e99a11 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 12:21:31 +0100 Subject: [PATCH 131/158] fix mypy errors --- nf_core/components/list.py | 61 ++++++++++++++++++++------------------ nf_core/create.py | 2 +- nf_core/lint/__init__.py | 44 ++++++++++++++------------- nf_core/synced_repo.py | 4 +-- requirements-dev.txt | 2 ++ 5 files changed, 61 insertions(+), 52 deletions(-) diff --git a/nf_core/components/list.py b/nf_core/components/list.py index d05c6d84a5..47c0eaad62 100644 --- a/nf_core/components/list.py +++ b/nf_core/components/list.py @@ -2,7 +2,7 @@ import logging from typing import Dict, List, Optional, Tuple, Union, cast -import rich +import rich.table from nf_core.components.components_command import ComponentCommand from nf_core.modules.modules_json import ModulesJson @@ -24,7 +24,7 @@ def __init__( super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull) self.remote = remote - def list_components(self, keywords: Optional[List[str]] = None, print_json=False) -> rich.table.Table: + def list_components(self, keywords: Optional[List[str]] = None, print_json=False) -> Union[rich.table.Table, str]: keywords = keywords or [] """ Get available modules/subworkflows names from GitHub tree for repo @@ -38,7 +38,7 @@ def list_components(self, keywords: Optional[List[str]] = None, print_json=False table.add_column(f"{self.component_type[:-1].capitalize()} Name") components: List[str] = [] - def pattern_msg(keywords: List[str]): + def pattern_msg(keywords: List[str]) -> str: if len(keywords) == 0: return "" if len(keywords) == 1: @@ -107,37 +107,40 @@ def pattern_msg(keywords: List[str]): table.add_column("Date") # Load 'modules.json' - modules_json = modules_json.modules_json + modules_json_file = modules_json.modules_json for repo_url, component_with_dir in sorted(repos_with_comps.items()): repo_entry: Dict[str, Dict[str, Dict[str, Dict[str, Union[str, List[str]]]]]] - - repo_entry = modules_json["repos"].get(repo_url, {}) - for install_dir, component in sorted(component_with_dir): - # Use cast() to predict the return type of recursive get():s - repo_modules = cast(dict, repo_entry.get(self.component_type)) - component_entry = cast(dict, cast(dict, repo_modules.get(install_dir)).get(component)) - - if component_entry: - version_sha = component_entry["git_sha"] - try: - # pass repo_name to get info on modules even outside nf-core/modules - message, date = ModulesRepo( - remote_url=repo_url, - branch=component_entry["branch"], - ).get_commit_info(version_sha) - except LookupError as e: - log.warning(e) + if modules_json_file is None: + log.warning(f"Modules JSON file '{modules_json.modules_json_path}' is missing. ") + continue + else: + repo_entry = modules_json_file["repos"].get(repo_url, {}) + for install_dir, component in sorted(component_with_dir): + # Use cast() to predict the return type of recursive get():s + repo_modules = cast(dict, repo_entry.get(self.component_type)) + component_entry = cast(dict, cast(dict, repo_modules.get(install_dir)).get(component)) + + if component_entry: + version_sha = component_entry["git_sha"] + try: + # pass repo_name to get info on modules even outside nf-core/modules + message, date = ModulesRepo( + remote_url=repo_url, + branch=component_entry["branch"], + ).get_commit_info(version_sha) + except LookupError as e: + log.warning(e) + date = "[red]Not Available" + message = "[red]Not Available" + else: + log.warning( + f"Commit SHA for {self.component_type[:-1]} '{install_dir}/{self.component_type}' is missing from 'modules.json'" + ) + version_sha = "[red]Not Available" date = "[red]Not Available" message = "[red]Not Available" - else: - log.warning( - f"Commit SHA for {self.component_type[:-1]} '{install_dir}/{self.component_type}' is missing from 'modules.json'" - ) - version_sha = "[red]Not Available" - date = "[red]Not Available" - message = "[red]Not Available" - table.add_row(component, repo_url, version_sha, message, date) + table.add_row(component, repo_url, version_sha, message, date) if print_json: return json.dumps(components, sort_keys=True, indent=4) diff --git a/nf_core/create.py b/nf_core/create.py index 470623f551..56d0912a07 100644 --- a/nf_core/create.py +++ b/nf_core/create.py @@ -11,7 +11,7 @@ import time from pathlib import Path -import filetype +import filetype # type: ignore import git import jinja2 import questionary diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 70f7ea925f..6db1274e95 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -164,26 +164,30 @@ class PipelineLint(nf_core.utils.Pipeline): warned (list): A list of tuples of the form: ``(, )`` """ - from .actions_awsfulltest import actions_awsfulltest - from .actions_awstest import actions_awstest - from .actions_ci import actions_ci - from .actions_schema_validation import actions_schema_validation - from .files_exist import files_exist - from .files_unchanged import files_unchanged - from .merge_markers import merge_markers - from .modules_json import modules_json - from .modules_structure import modules_structure - from .multiqc_config import multiqc_config - from .nextflow_config import nextflow_config - from .pipeline_name_conventions import pipeline_name_conventions - from .pipeline_todos import pipeline_todos - from .readme import readme - from .schema_description import schema_description - from .schema_lint import schema_lint - from .schema_params import schema_params - from .system_exit import system_exit - from .template_strings import template_strings - from .version_consistency import version_consistency + from .actions_awsfulltest import actions_awsfulltest # type: ignore[misc] + from .actions_awstest import actions_awstest # type: ignore[misc] + from .actions_ci import actions_ci # type: ignore[misc] + from .actions_schema_validation import ( + actions_schema_validation, # type: ignore[misc] + ) + from .files_exist import files_exist # type: ignore[misc] + from .files_unchanged import files_unchanged # type: ignore[misc] + from .merge_markers import merge_markers # type: ignore[misc] + from .modules_json import modules_json # type: ignore[misc] + from .modules_structure import modules_structure # type: ignore[misc] + from .multiqc_config import multiqc_config # type: ignore[misc] + from .nextflow_config import nextflow_config # type: ignore[misc] + from .pipeline_name_conventions import ( + pipeline_name_conventions, # type: ignore[misc] + ) + from .pipeline_todos import pipeline_todos # type: ignore[misc] + from .readme import readme # type: ignore[misc] + from .schema_description import schema_description # type: ignore[misc] + from .schema_lint import schema_lint # type: ignore[misc] + from .schema_params import schema_params # type: ignore[misc] + from .system_exit import system_exit # type: ignore[misc] + from .template_strings import template_strings # type: ignore[misc] + from .version_consistency import version_consistency # type: ignore[misc] def __init__( self, wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False diff --git a/nf_core/synced_repo.py b/nf_core/synced_repo.py index 41e0853f2e..a2107f633c 100644 --- a/nf_core/synced_repo.py +++ b/nf_core/synced_repo.py @@ -3,9 +3,9 @@ import os import shutil from pathlib import Path +from typing import Dict import git -import rich import rich.progress from git.exc import GitCommandError @@ -61,7 +61,7 @@ class SyncedRepo: An object to store details about a locally cached code repository. """ - local_repo_statuses = {} + local_repo_statuses: Dict[str, bool] = {} no_pull_global = False @staticmethod diff --git a/requirements-dev.txt b/requirements-dev.txt index c94874b193..12183eb98c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,3 +9,5 @@ sphinx-rtd-theme mypy types-PyYAML types-requests +types-jsonschema +types-Markdown From 341dbdf2bf0ec5b8b5642c5422b565f26c323477 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 12:55:07 +0100 Subject: [PATCH 132/158] fix type comments --- nf_core/lint/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 6db1274e95..b23f957d2a 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -167,8 +167,8 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_awsfulltest import actions_awsfulltest # type: ignore[misc] from .actions_awstest import actions_awstest # type: ignore[misc] from .actions_ci import actions_ci # type: ignore[misc] - from .actions_schema_validation import ( - actions_schema_validation, # type: ignore[misc] + from .actions_schema_validation import ( # type: ignore[misc] + actions_schema_validation, ) from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] @@ -177,8 +177,8 @@ class PipelineLint(nf_core.utils.Pipeline): from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] from .nextflow_config import nextflow_config # type: ignore[misc] - from .pipeline_name_conventions import ( - pipeline_name_conventions, # type: ignore[misc] + from .pipeline_name_conventions import ( # type: ignore[misc] + pipeline_name_conventions, ) from .pipeline_todos import pipeline_todos # type: ignore[misc] from .readme import readme # type: ignore[misc] From 295a44aba893cc7f430670e334e486d8265475ff Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 27 Nov 2023 14:06:55 +0100 Subject: [PATCH 133/158] fix: use double quotes for interpretable string --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index a2cabfd2f4..9b94938433 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -27,7 +27,7 @@ process {{ component_name_underscore|upper }} { // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. {% endif -%} - conda '${modulesDir}/environment.yml' + conda "${modulesDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'biocontainers/YOUR-TOOL-HERE' }}' }" From 7a4ddfce68f9892b44ed1cb0aac856f30bf36abb Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 27 Nov 2023 14:07:18 +0100 Subject: [PATCH 134/158] fix: correct implicit variable name https://www.nextflow.io/docs/latest/script.html#implicit-variables --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 9b94938433..d482fa0b43 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -27,7 +27,7 @@ process {{ component_name_underscore|upper }} { // For Conda, the build (i.e. "h9402c20_2") must be EXCLUDED to support installation on different operating systems. // TODO nf-core: See section in main README for further information regarding finding and adding container addresses to the section below. {% endif -%} - conda "${modulesDir}/environment.yml" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? '{{ singularity_container if singularity_container else 'https://depot.galaxyproject.org/singularity/YOUR-TOOL-HERE' }}': '{{ docker_container if docker_container else 'biocontainers/YOUR-TOOL-HERE' }}' }" From 1faff58f62aab008652d3659692e234d83fc5ce4 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 27 Nov 2023 14:12:20 +0100 Subject: [PATCH 135/158] docs: describe fix in changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f64047db4b..6ae69fc7ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Fix writing files to a remote outdir in the NfcoreTemplate helper functions ([#2465](https://github.com/nf-core/tools/pull/2465)) - Fancier syntax highlighting for example samplesheets in the usage.md template ([#2503](https://github.com/nf-core/tools/pull/2503)) - Use closure for multiqc ext.args ([#2509](https://github.com/nf-core/tools/pull/2509)) +- Fix how the modules template references the conda environment file ([#2540](https://github.com/nf-core/tools/pull/2540)) ### Linting From 8f9b8515c55e4d88a1c053f3c7d56588c2be23f1 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 15:38:27 +0100 Subject: [PATCH 136/158] split jobs up to only run once on dev --- .github/workflows/pytest.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0fc7f3d094..4d5853499b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -24,7 +24,7 @@ env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} jobs: - pytest: + setup: runs-on: ${{ matrix.runner }} strategy: matrix: @@ -34,16 +34,32 @@ jobs: - runner: "ubuntu-20.04" python-version: "3.8" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 name: Check out source-code repository - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Check conditions + id: conditions + run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> $GITHUB_ENV + + test: + needs: setup + if: ${{ needs.setup.outputs.run-tests == 'true' }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v2 + name: Check out source-code repository + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: "pip" - # run full test matrix only on PRs to the master branch - if: github.ref == 'refs/heads/master' || matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8' - name: Install dependencies run: | From 442b17a0e6eedc4f2675e6981c78d72c06958ae8 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 15:44:29 +0100 Subject: [PATCH 137/158] check condition before setting up anything --- .github/workflows/pytest.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4d5853499b..6886656ca9 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -33,19 +33,22 @@ jobs: include: - runner: "ubuntu-20.04" python-version: "3.8" + steps: - - uses: actions/checkout@v2 - name: Check out source-code repository + - name: Check conditions + id: conditions + run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> $GITHUB_ENV + + - name: Check out source-code repository + uses: actions/checkout@v4 + if: ${{ needs.conditions.outputs.run-tests == 'true' }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} cache: "pip" - - - name: Check conditions - id: conditions - run: echo "run-tests=${{ github.ref == 'refs/heads/master' || (matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8') }}" >> $GITHUB_ENV + if: ${{ needs.conditions.outputs.run-tests == 'true' }} test: needs: setup From 26cc37b0e628dbcd02021025abaf71a1adfdd49b Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 15:49:55 +0100 Subject: [PATCH 138/158] fix copy-paste error --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 6886656ca9..cd0d58530e 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -44,7 +44,7 @@ jobs: if: ${{ needs.conditions.outputs.run-tests == 'true' }} - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: "pip" From f54b9f26b966d78a982d37705fbc4b851e641e5d Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 16:18:02 +0100 Subject: [PATCH 139/158] fix variable name in lint test --- nf_core/modules/lint/environment_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index d9c0cbf7f9..1367054c8d 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -32,7 +32,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) # check if the module's main.nf requires a conda environment with open(Path(module.component_dir, "main.nf"), "r") as fh: main_nf = fh.read() - if "conda '${modulesDir}/environment.yml'" in main_nf: + if "conda '${moduleDir}/environment.yml'" in main_nf: module.failed.append( ("environment_yml_exists", "Module's `environment.yml` does not exist", module.environment_yml) ) From 36ec7930c6de9bc7fdda7313a3a0befc1e2a8ae0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Mon, 27 Nov 2023 16:36:47 +0100 Subject: [PATCH 140/158] fix incorrectly guessed syntax --- .github/workflows/pytest.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index cd0d58530e..340768b832 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -41,14 +41,16 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v4 - if: ${{ needs.conditions.outputs.run-tests == 'true' }} + if: ${{ steps.conditions.outputs.run-tests == 'true' }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: "pip" - if: ${{ needs.conditions.outputs.run-tests == 'true' }} + if: ${{ steps.conditions.outputs.run-tests == 'true' }} + outputs: + run-tests: ${{ env.run-tests }} test: needs: setup From 84f92bc09c1ba167403656c32b2046402bef5079 Mon Sep 17 00:00:00 2001 From: "Moritz E. Beber" Date: Mon, 27 Nov 2023 18:52:23 +0100 Subject: [PATCH 141/158] fix: adjust quotes to match template --- nf_core/modules/lint/environment_yml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/environment_yml.py b/nf_core/modules/lint/environment_yml.py index 1367054c8d..a052425539 100644 --- a/nf_core/modules/lint/environment_yml.py +++ b/nf_core/modules/lint/environment_yml.py @@ -32,7 +32,7 @@ def environment_yml(module_lint_object: ComponentLint, module: NFCoreComponent) # check if the module's main.nf requires a conda environment with open(Path(module.component_dir, "main.nf"), "r") as fh: main_nf = fh.read() - if "conda '${moduleDir}/environment.yml'" in main_nf: + if 'conda "${moduleDir}/environment.yml"' in main_nf: module.failed.append( ("environment_yml_exists", "Module's `environment.yml` does not exist", module.environment_yml) ) From d2c5400877b2458a9c7faeef183f553cd64f05d9 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 14:17:14 +0100 Subject: [PATCH 142/158] add mypy to pre-commit config, fix mypy errors --- .pre-commit-config.yaml | 10 +++++ docs/api/_src/conf.py | 5 ++- mypy.ini | 1 + nf_core/__main__.py | 56 +++++++++++++-------------- nf_core/params_file.py | 2 +- nf_core/subworkflows/lint/__init__.py | 12 +++--- pyproject.toml | 5 --- requirements-dev.txt | 1 + tests/test_lint.py | 20 +++++----- 9 files changed, 58 insertions(+), 54 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7aeeb5bc9..578d17f769 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,3 +11,13 @@ repos: rev: "v2.7.1" hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.7.1" # Use the sha / tag you want to point at + hooks: + - id: mypy + additional_dependencies: + - types-PyYAML + - types-requests + - types-jsonschema + - types-Markdown + - types-setuptools diff --git a/docs/api/_src/conf.py b/docs/api/_src/conf.py index 4d8ae661d5..27eaf9bcb3 100644 --- a/docs/api/_src/conf.py +++ b/docs/api/_src/conf.py @@ -14,6 +14,7 @@ # import os import sys +from typing import Dict sys.path.insert(0, os.path.abspath("../../../nf_core")) import nf_core @@ -58,7 +59,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language: str = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -113,7 +114,7 @@ # -- Options for LaTeX output ------------------------------------------------ -latex_elements = { +latex_elements: Dict[str, str] = { # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', diff --git a/mypy.ini b/mypy.ini index f869fa7e47..c48aa5884b 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] warn_unused_configs = True +ignore_missing_imports = true diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 232f1bf116..ec5ddeeaa8 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -151,7 +151,7 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): # nf-core list -@nf_core_cli.command() +@nf_core_cli.command("list") @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option( "-s", @@ -162,7 +162,7 @@ def nf_core_cli(ctx, verbose, hide_progress, log_file): ) @click.option("--json", is_flag=True, default=False, help="Print full output as JSON") @click.option("--show-archived", is_flag=True, default=False, help="Print archived workflows") -def list(keywords, sort, json, show_archived): +def list_pipelines(keywords, sort, json, show_archived): """ List available nf-core pipelines with local info. @@ -553,9 +553,9 @@ def subworkflows(ctx, git_remote, branch, no_pull): # nf-core modules list subcommands -@modules.group() +@modules.group("list") @click.pass_context -def list(ctx): +def modules_list(ctx): """ List modules in a local pipeline or remote repository. """ @@ -563,11 +563,11 @@ def list(ctx): # nf-core modules list remote -@list.command() +@modules_list.command("remote") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): +def modules_list_remote(ctx, keywords, json): """ List modules in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -588,7 +588,7 @@ def remote(ctx, keywords, json): # nf-core modules list local -@list.command() +@modules_list.command("local") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") @@ -599,7 +599,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin +def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List modules installed locally in a pipeline """ @@ -620,7 +620,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core modules install -@modules.command() +@modules.command("install") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -633,7 +633,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin @click.option("-p", "--prompt", is_flag=True, default=False, help="Prompt for the version of the module") @click.option("-f", "--force", is_flag=True, default=False, help="Force reinstallation of module if it already exists") @click.option("-s", "--sha", type=str, metavar="", help="Install module at commit SHA") -def install(ctx, tool, dir, prompt, force, sha): +def modules_install(ctx, tool, dir, prompt, force, sha): """ Install DSL2 modules within a pipeline. @@ -660,7 +660,7 @@ def install(ctx, tool, dir, prompt, force, sha): # nf-core modules update -@modules.command() +@modules.command("update") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -699,7 +699,7 @@ def install(ctx, tool, dir, prompt, force, sha): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps): +def modules_update(ctx, tool, directory, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 modules within a pipeline. @@ -767,7 +767,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove -@modules.command() +@modules.command("remove") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -777,7 +777,7 @@ def patch(ctx, tool, dir, remove): default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) -def remove(ctx, dir, tool): +def modules_remove(ctx, dir, tool): """ Remove a module from a pipeline. """ @@ -887,7 +887,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once): # nf-core modules lint -@modules.command() +@modules.command("lint") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -912,9 +912,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once): show_default=True, ) @click.option("--fix-version", is_flag=True, help="Fix the module version if a newer version is available") -def lint( - ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version -): # pylint: disable=redefined-outer-name +def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, sort_by, fix_version): """ Lint one or more modules in a directory. @@ -959,7 +957,7 @@ def lint( # nf-core modules info -@modules.command() +@modules.command("info") @click.pass_context @click.argument("tool", type=str, required=False, metavar=" or ") @click.option( @@ -969,7 +967,7 @@ def lint( default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def info(ctx, tool, dir): +def modules_info(ctx, tool, dir): """ Show developer usage information about a given module. @@ -1095,9 +1093,9 @@ def test_subworkflow(ctx, subworkflow, dir, no_prompts, update, once): # nf-core subworkflows list subcommands -@subworkflows.group() +@subworkflows.group("list") @click.pass_context -def list(ctx): +def subworkflows_list(ctx): """ List subworkflows in a local pipeline or remote repository. """ @@ -1105,7 +1103,7 @@ def list(ctx): # nf-core subworkflows list remote -@list.command() +@subworkflows_list.command("remote") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") @@ -1131,7 +1129,7 @@ def remote(ctx, keywords, json): # nf-core subworkflows list local -@list.command() +@subworkflows_list.command("local") @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") @@ -1163,7 +1161,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin # nf-core subworkflows lint -@subworkflows.command() +@subworkflows.command("lint") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") @@ -1187,9 +1185,7 @@ def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin help="Sort lint output by subworkflow or test name.", show_default=True, ) -def lint( - ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by -): # pylint: disable=redefined-outer-name +def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, local, passed, sort_by): """ Lint one or more subworkflows in a directory. @@ -1496,11 +1492,11 @@ def build(dir, no_prompts, web_only, url): # nf-core schema lint -@schema.command() +@schema.command("lint") @click.argument( "schema_path", type=click.Path(exists=True), default="nextflow_schema.json", metavar="" ) -def lint(schema_path): +def schema_lint(schema_path): """ Check that a given pipeline schema is valid. diff --git a/nf_core/params_file.py b/nf_core/params_file.py index 39986b95c2..5c50c53fb9 100644 --- a/nf_core/params_file.py +++ b/nf_core/params_file.py @@ -89,7 +89,7 @@ def __init__( self, pipeline=None, revision=None, - ): + ) -> None: """Initialise the ParamFileBuilder class Args: diff --git a/nf_core/subworkflows/lint/__init__.py b/nf_core/subworkflows/lint/__init__.py index 44c7c21a37..ffba41f9da 100644 --- a/nf_core/subworkflows/lint/__init__.py +++ b/nf_core/subworkflows/lint/__init__.py @@ -29,12 +29,12 @@ class SubworkflowLint(ComponentLint): """ # Import lint functions - from .main_nf import main_nf - from .meta_yml import meta_yml - from .subworkflow_changes import subworkflow_changes - from .subworkflow_tests import subworkflow_tests - from .subworkflow_todos import subworkflow_todos - from .subworkflow_version import subworkflow_version + from .main_nf import main_nf # type: ignore[misc] + from .meta_yml import meta_yml # type: ignore[misc] + from .subworkflow_changes import subworkflow_changes # type: ignore[misc] + from .subworkflow_tests import subworkflow_tests # type: ignore[misc] + from .subworkflow_todos import subworkflow_todos # type: ignore[misc] + from .subworkflow_version import subworkflow_version # type: ignore[misc] def __init__( self, diff --git a/pyproject.toml b/pyproject.toml index f0702742fd..2380073107 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,3 @@ norecursedirs = [ ".*", "build", "dist", "*.egg", "data", "__pycache__", ".githu profile = "black" known_first_party = ["nf_core"] multi_line_output = 3 - -[tool.mypy] -ignore_missing_imports = true -follow_imports = "skip" -disable_error_code = "no-redef" diff --git a/requirements-dev.txt b/requirements-dev.txt index 12183eb98c..3ef0593085 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -11,3 +11,4 @@ types-PyYAML types-requests types-jsonschema types-Markdown +types-setuptools diff --git a/tests/test_lint.py b/tests/test_lint.py index 67104e3ad0..69917b279d 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -178,44 +178,44 @@ def test_sphinx_md_files(self): ####################### # SPECIFIC LINT TESTS # ####################### - from .lint.actions_awsfulltest import ( + from .lint.actions_awsfulltest import ( # type: ignore[misc] test_actions_awsfulltest_fail, test_actions_awsfulltest_pass, test_actions_awsfulltest_warn, ) - from .lint.actions_awstest import ( + from .lint.actions_awstest import ( # type: ignore[misc] test_actions_awstest_fail, test_actions_awstest_pass, ) - from .lint.actions_ci import ( + from .lint.actions_ci import ( # type: ignore[misc] test_actions_ci_fail_wrong_nf, test_actions_ci_fail_wrong_trigger, test_actions_ci_pass, ) - from .lint.actions_schema_validation import ( + from .lint.actions_schema_validation import ( # type: ignore[misc] test_actions_schema_validation_fails_for_additional_property, test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) - from .lint.files_exist import ( + from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_missing_config, test_files_exist_missing_main, test_files_exist_pass, ) - from .lint.files_unchanged import ( + from .lint.files_unchanged import ( # type: ignore[misc] test_files_unchanged_fail, test_files_unchanged_pass, ) - from .lint.merge_markers import test_merge_markers_found - from .lint.modules_json import test_modules_json_pass - from .lint.nextflow_config import ( + from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] + from .lint.modules_json import test_modules_json_pass # type: ignore[misc] + from .lint.nextflow_config import ( # type: ignore[misc] test_nextflow_config_bad_name_fail, test_nextflow_config_dev_in_release_mode_failed, test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) - from .lint.version_consistency import test_version_consistency + from .lint.version_consistency import test_version_consistency # type: ignore[misc] # TODO nf-core: Assess and strip out if no longer required for DSL2 From 62ee35e4d44339569206f79437a3a5076b7db175 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 14:37:52 +0100 Subject: [PATCH 143/158] try different condition and without accessing the matrix --- .github/workflows/pytest.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 340768b832..03d5bc65e5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -41,29 +41,31 @@ jobs: - name: Check out source-code repository uses: actions/checkout@v4 - if: ${{ steps.conditions.outputs.run-tests == 'true' }} + if: ${{ env.run-tests == 'true' }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} cache: "pip" - if: ${{ steps.conditions.outputs.run-tests == 'true' }} + if: ${{ env.run-tests == 'true' }} outputs: + python-version: ${{ matrix.python-version }} + runner: ${{ matrix.runner }} run-tests: ${{ env.run-tests }} test: needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ${{ matrix.runner }} + runs-on: ${{ needs.setup.outputs.runner }} steps: - uses: actions/checkout@v2 name: Check out source-code repository - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python ${{ needs.setup.outputs.python-version }} uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" - name: Install dependencies @@ -72,7 +74,7 @@ jobs: pip install -e . - name: Downgrade git to the Ubuntu official repository's version - if: ${{ matrix.runner == 'ubuntu-20.04' && matrix.python-version == '3.8' }} + if: ${{ needs.setup.outputs.runner == 'ubuntu-20.04' && needs.setup.outputs.python-version == '3.8' }} run: | sudo apt update sudo apt remove git git-man From b442d50adc5ec9b87ca045eb23dca4be5b5d9212 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 15:01:11 +0100 Subject: [PATCH 144/158] use same naming for modules and subworkflo commands --- nf_core/__main__.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index ec5ddeeaa8..43095bc194 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1107,7 +1107,7 @@ def subworkflows_list(ctx): @click.pass_context @click.argument("keywords", required=False, nargs=-1, metavar="") @click.option("-j", "--json", is_flag=True, help="Print as JSON to stdout") -def remote(ctx, keywords, json): +def subworkflows_list_remote(ctx, keywords, json): """ List subworkflows in a remote GitHub repo [dim i](e.g [link=https://github.com/nf-core/modules]nf-core/modules[/])[/]. """ @@ -1140,7 +1140,7 @@ def remote(ctx, keywords, json): default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin +def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-builtin """ List subworkflows installed locally in a pipeline """ @@ -1229,7 +1229,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info -@subworkflows.command() +@subworkflows.command("info") @click.pass_context @click.argument("tool", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1239,7 +1239,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def info(ctx, tool, dir): +def subworkflows_info(ctx, tool, dir): """ Show developer usage information about a given subworkflow. @@ -1268,7 +1268,7 @@ def info(ctx, tool, dir): # nf-core subworkflows install -@subworkflows.command() +@subworkflows.command("install") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1283,7 +1283,7 @@ def info(ctx, tool, dir): "-f", "--force", is_flag=True, default=False, help="Force reinstallation of subworkflow if it already exists" ) @click.option("-s", "--sha", type=str, metavar="", help="Install subworkflow at commit SHA") -def install(ctx, subworkflow, dir, prompt, force, sha): +def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): """ Install DSL2 subworkflow within a pipeline. @@ -1310,7 +1310,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove -@subworkflows.command() +@subworkflows.command("remove") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1320,7 +1320,7 @@ def install(ctx, subworkflow, dir, prompt, force, sha): default=".", help=r"Pipeline directory. [dim]\[default: current working directory][/]", ) -def remove(ctx, dir, subworkflow): +def subworkflows_remove(ctx, dir, subworkflow): """ Remove a subworkflow from a pipeline. """ @@ -1340,7 +1340,7 @@ def remove(ctx, dir, subworkflow): # nf-core subworkflows update -@subworkflows.command() +@subworkflows.command("update") @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option( @@ -1378,7 +1378,7 @@ def remove(ctx, dir, subworkflow): default=False, help="Automatically update all linked modules and subworkflows without asking for confirmation", ) -def update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): +def subworkflows_update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save_diff, update_deps): """ Update DSL2 subworkflow within a pipeline. @@ -1392,7 +1392,7 @@ def update(ctx, subworkflow, dir, force, prompt, sha, install_all, preview, save force, prompt, sha, - all, + install_all, preview, save_diff, update_deps, From ed8efbcea8e0b13d508c9229c82fc8412a3acc63 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 15:53:34 +0100 Subject: [PATCH 145/158] update github checkout and setup-python --- .github/workflows/create-lint-wf.yml | 4 ++-- .../workflows/create-test-lint-wf-template.yml | 4 ++-- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/deploy-pypi.yml | 2 +- .github/workflows/fix-linting.yml | 4 ++-- .github/workflows/lint-code.yml | 16 ++++++++-------- .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/push_dockerhub_release.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/rich-codex.yml | 2 +- .github/workflows/sync.yml | 4 ++-- .github/workflows/tools-api-docs-dev.yml | 2 +- .github/workflows/tools-api-docs-release.yml | 2 +- CHANGELOG.md | 1 + .../pipeline-template/.github/workflows/ci.yml | 2 +- .../.github/workflows/fix-linting.yml | 4 ++-- .../.github/workflows/linting.yml | 12 ++++++------ 17 files changed, 34 insertions(+), 33 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index d5f377a486..51daec21b7 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -27,7 +27,7 @@ jobs: - "latest-everything" steps: # Get the repo code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository # Set up nf-core/tools @@ -48,7 +48,7 @@ jobs: version: ${{ matrix.NXF_VER }} # Install the Prettier linting tools - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 5ce9b41118..f1c6c4d149 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -33,7 +33,7 @@ jobs: - "template_skip_nf_core_configs.yml" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 @@ -52,7 +52,7 @@ jobs: version: latest-everything # Install the Prettier linting tools - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index d96518d7c6..4f66adae6b 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -26,7 +26,7 @@ jobs: - "23.04.0" - "latest-everything" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index 9b46f0731b..b847df9218 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python 3.12 diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index f56cb927fc..a6cf149569 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index a951230a6a..baaaedcb4a 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -16,9 +16,9 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -30,9 +30,9 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @@ -43,7 +43,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check code lints with Black uses: psf/black@stable @@ -74,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 @@ -89,8 +89,8 @@ jobs: static-type-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 with: python-version: 3.12 cache: "pip" diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index dea28cdd35..169a917d83 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 857b241022..49ce17dd84 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0fc7f3d094..be9942dd16 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -34,7 +34,7 @@ jobs: - runner: "ubuntu-20.04" python-version: "3.8" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out source-code repository - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 54aaf240df..8368255390 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 5e7719973f..b8fb287944 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -36,10 +36,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out nf-core/tools - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 3a4725a944..7de8913e03 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 8a6eda1318..412784c233 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -19,7 +19,7 @@ jobs: - ${{ github.event.release.tag_name }} steps: - name: Check out source-code repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.12 uses: actions/setup-python@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae69fc7ab..0ab328866f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) - Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)). +- Add mypy to pre-commit config for the tools repo ([#2545](https://github.com/nf-core/tools/pull/2545)) # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 521f3e664a..3edd49f09d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index f3dc3e50fe..31e8cd2b36 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -24,7 +24,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index edce89065f..64d1851f2e 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,9 +14,9 @@ jobs: EditorConfig: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -27,9 +27,9 @@ jobs: Prettier: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 - name: Install Prettier run: npm install -g prettier @@ -40,7 +40,7 @@ jobs: PythonBlack: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check code lints with Black uses: psf/black@stable @@ -71,7 +71,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 From ae505c46cd271582c50aa956bfbc9449222bcab4 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 16:01:22 +0100 Subject: [PATCH 146/158] =?UTF-8?q?switch=20to=20self-hosted=20?= =?UTF-8?q?=F0=9F=A4=9E=F0=9F=8F=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 03d5bc65e5..b6edac5182 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -57,7 +57,7 @@ jobs: test: needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ${{ needs.setup.outputs.runner }} + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v2 name: Check out source-code repository From 1ad27a7eb955a13dcc5dc53ab576be94c7429126 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 16:05:26 +0100 Subject: [PATCH 147/158] =?UTF-8?q?Revert=20"switch=20to=20self-hosted=20?= =?UTF-8?q?=F0=9F=A4=9E=F0=9F=8F=BB"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit ae505c46cd271582c50aa956bfbc9449222bcab4. --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b6edac5182..03d5bc65e5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -57,7 +57,7 @@ jobs: test: needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ["self-hosted"] + runs-on: ${{ needs.setup.outputs.runner }} steps: - uses: actions/checkout@v2 name: Check out source-code repository From 13bbac6570af595697aaec638a82d7e7fd4fa3ac Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 16:32:57 +0100 Subject: [PATCH 148/158] fix subworkflow command --- nf_core/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 232f1bf116..81e7de0f16 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -1064,7 +1064,6 @@ def create_subworkflow(ctx, subworkflow, dir, author, force): @click.pass_context @click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") @click.option("-d", "--dir", type=click.Path(exists=True), default=".", metavar="") -@click.option("-t", "--run-tests", is_flag=True, default=False, help="Run the test workflows") @click.option("-p", "--no-prompts", is_flag=True, default=False, help="Use defaults without prompting") @click.option("-u", "--update", is_flag=True, default=False, help="Update existing snapshots") @click.option("-o", "--once", is_flag=True, default=False, help="Run tests only once. Don't check snapshot stability") From ad03acdd8007e482f7f6d1df2dd9b3b9394ab103 Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 28 Nov 2023 22:00:46 +0100 Subject: [PATCH 149/158] fix parsing for include statement of nested subworkflows, print long messages --- nf_core/components/lint/__init__.py | 17 ++++++++++++----- nf_core/components/nfcore_component.py | 13 ++++++++++--- nf_core/subworkflows/lint/subworkflow_tests.py | 1 + 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/nf_core/components/lint/__init__.py b/nf_core/components/lint/__init__.py index c46d68ff22..efffc28e85 100644 --- a/nf_core/components/lint/__init__.py +++ b/nf_core/components/lint/__init__.py @@ -10,7 +10,9 @@ import os from pathlib import Path -import rich +import rich.box +import rich.console +import rich.panel from rich.markdown import Markdown from rich.table import Table @@ -209,7 +211,7 @@ def _print_results(self, show_passed=False, sort_by="test"): self.failed.sort(key=operator.attrgetter(*sort_order)) # Find maximum module name length - max_name_len = 40 + max_name_len = len(self.component_type[:-1] + " name") for tests in [self.passed, self.warned, self.failed]: try: for lint_result in tests: @@ -264,7 +266,7 @@ def format_result(test_results, table): table = Table(style="yellow", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len) table.add_column("File path") - table.add_column("Test message") + table.add_column("Test message", overflow="fold") table = format_result(self.warned, table) console.print( rich.panel.Panel( @@ -278,10 +280,15 @@ def format_result(test_results, table): # Table of failing tests if len(self.failed) > 0: - table = Table(style="red", box=rich.box.MINIMAL, pad_edge=False, border_style="dim") + table = Table( + style="red", + box=rich.box.MINIMAL, + pad_edge=False, + border_style="dim", + ) table.add_column(f"{self.component_type[:-1].title()} name", width=max_name_len) table.add_column("File path") - table.add_column("Test message") + table.add_column("Test message", overflow="fold") table = format_result(self.failed, table) console.print( rich.panel.Panel( diff --git a/nf_core/components/nfcore_component.py b/nf_core/components/nfcore_component.py index 190310faae..8d9b6840b0 100644 --- a/nf_core/components/nfcore_component.py +++ b/nf_core/components/nfcore_component.py @@ -4,6 +4,7 @@ import logging import re from pathlib import Path +from typing import Union log = logging.getLogger(__name__) @@ -77,7 +78,7 @@ def __init__( self.test_yml = None self.test_main_nf = None - def _get_main_nf_tags(self, test_main_nf: str): + def _get_main_nf_tags(self, test_main_nf: Union[Path, str]): """Collect all tags from the main.nf.test file.""" tags = [] with open(test_main_nf, "r") as fh: @@ -86,13 +87,19 @@ def _get_main_nf_tags(self, test_main_nf: str): tags.append(line.strip().split()[1].strip('"')) return tags - def _get_included_components(self, main_nf: str): + def _get_included_components(self, main_nf: Union[Path, str]): """Collect all included components from the main.nf file.""" included_components = [] with open(main_nf, "r") as fh: for line in fh: if line.strip().startswith("include"): - included_components.append(line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/")) + # get tool/subtool or subworkflow name from include statement, can be in the form + #'../../../modules/nf-core/hisat2/align/main' + #'../bam_sort_stats_samtools/main' + #'../subworkflows/nf-core/bam_sort_stats_samtools/main' + component = line.strip().split()[-1].split(self.org)[-1].split("main")[0].strip("/") + component = component.replace("'../", "subworkflows/") + included_components.append(component) return included_components def get_inputs_from_main_nf(self): diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index e82d00f636..30439a8d0b 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -107,6 +107,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): included_components = [] if subworkflow.main_nf.is_file(): included_components = subworkflow._get_included_components(subworkflow.main_nf) + log.debug(f"Required tags: {required_tags}") missing_tags = [] for tag in required_tags + included_components: if tag not in main_nf_tags: From da7636a3e44ead54ffacce02f20dd175cc06121a Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 13:25:03 +0100 Subject: [PATCH 150/158] fix subworkflow tag in template --- nf_core/subworkflow-template/subworkflows/tests/main.nf.test | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test index b0a212f2a3..59c4fdb679 100644 --- a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test +++ b/nf_core/subworkflow-template/subworkflows/tests/main.nf.test @@ -8,7 +8,6 @@ nextflow_workflow { tag "subworkflows" tag "subworkflows_nfcore" - tag "{{ component_name }}" tag "subworkflows/{{ component_name }}" // TODO nf-core: Add tags for all modules used within this subworkflow. Example: tag "samtools" From cc3b6e1b9b15fb2b2bbbcb1bcd8eb521669e0182 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 15:59:14 +0100 Subject: [PATCH 151/158] fix linting tests --- .github/workflows/pytest.yml | 1 + .../subworkflows/lint/subworkflow_tests.py | 1 - tests/subworkflows/lint.py | 11 +++++++---- tests/test_subworkflows.py | 19 ++++++++----------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 03d5bc65e5..2557c4fa4f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -55,6 +55,7 @@ jobs: run-tests: ${{ env.run-tests }} test: + name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} runs-on: ${{ needs.setup.outputs.runner }} diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 30439a8d0b..a125be152f 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -102,7 +102,6 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): "subworkflows", f"subworkflows/{subworkflow.component_name}", "subworkflows_nfcore", - subworkflow.component_name, ] included_components = [] if subworkflow.main_nf.is_file(): diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index 5bbe746f2e..1380db2260 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -28,9 +28,8 @@ def test_subworkflows_lint_new_subworkflow(self): """lint a new subworkflow""" subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=True, all_subworkflows=True) - assert ( - len(subworkflow_lint.failed) == 1 # test snap missing after creating a subworkflow - ), f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -68,7 +67,6 @@ def test_subworkflows_lint_multiple_remotes(self): def test_subworkflows_lint_snapshot_file(self): """Test linting a subworkflow with a snapshot file""" - Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" @@ -78,8 +76,10 @@ def test_subworkflows_lint_snapshot_file(self): def test_subworkflows_lint_snapshot_file_missing_fail(self): """Test linting a subworkflow with a snapshot file missing, which should fail""" + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 @@ -96,8 +96,11 @@ def test_subworkflows_lint_snapshot_file_not_needed(self): Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test"), "w" ) as fh: fh.write(new_content) + + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").unlink() subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" assert len(subworkflow_lint.passed) > 0 assert len(subworkflow_lint.warned) >= 0 diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 33cad81e3d..a4ab959fb4 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -4,6 +4,7 @@ import os import shutil import unittest +from pathlib import Path import nf_core.create import nf_core.modules @@ -21,22 +22,18 @@ def create_modules_repo_dummy(tmp_dir): """Create a dummy copy of the nf-core/modules repo""" - root_dir = os.path.join(tmp_dir, "modules") - os.makedirs(os.path.join(root_dir, "modules")) - os.makedirs(os.path.join(root_dir, "subworkflows")) - os.makedirs(os.path.join(root_dir, "subworkflows", "nf-core")) - os.makedirs(os.path.join(root_dir, "tests", "modules")) - os.makedirs(os.path.join(root_dir, "tests", "subworkflows")) - os.makedirs(os.path.join(root_dir, "tests", "config")) - with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: - fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + root_dir = Path(tmp_dir, "modules") + Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) + Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) + Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) + Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) + with open(Path(root_dir, ".nf-core.yml"), "w") as fh: fh.writelines(["repository_type: modules", "\n", "org_path: nf-core", "\n"]) - # TODO Add a mock here subworkflow_create = nf_core.subworkflows.SubworkflowCreate(root_dir, "test_subworkflow", "@author", True) subworkflow_create.create() + Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap").touch() return root_dir From c6ff441dca15da1162d2018ea063dc76d4073dc5 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 16:13:04 +0100 Subject: [PATCH 152/158] try once more on self hosted --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2557c4fa4f..573c1364b6 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -58,7 +58,7 @@ jobs: name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ${{ needs.setup.outputs.runner }} + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v2 name: Check out source-code repository @@ -80,7 +80,7 @@ jobs: sudo apt update sudo apt remove git git-man sudo add-apt-repository --remove ppa:git-core/ppa - sudo apt install git + sudo apt install -y git - name: Get current date id: date run: echo "date=$(date +'%Y-%m')" >> $GITHUB_ENV From ffba25fa46cda05931e72f0a6cfcabe9a131f099 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 16:22:32 +0100 Subject: [PATCH 153/158] run even more things on aws --- .github/workflows/lint-code.yml | 10 +++++----- .github/workflows/pytest.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index baaaedcb4a..b348496c3c 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -14,7 +14,7 @@ concurrency: jobs: EditorConfig: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v4 @@ -28,7 +28,7 @@ jobs: run: editorconfig-checker -exclude README.md $(git ls-files | grep -v 'test\|.py\|md\|json\|yml\|yaml\|html\|css\|Makefile') Prettier: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v4 @@ -41,7 +41,7 @@ jobs: run: prettier --check ${GITHUB_WORKSPACE} PythonBlack: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v4 @@ -71,7 +71,7 @@ jobs: allow-repeats: false isort: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - name: Check out source-code repository uses: actions/checkout@v4 @@ -87,7 +87,7 @@ jobs: requirementsFiles: "requirements.txt requirements-dev.txt" static-type-check: - runs-on: ubuntu-latest + runs-on: ["self-hosted"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 573c1364b6..4f370bc280 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -25,7 +25,7 @@ env: jobs: setup: - runs-on: ${{ matrix.runner }} + runs-on: ["self-hosted"] strategy: matrix: python-version: ["3.8", "3.12"] From d64c3b9aa5e1a6523f5e2c5ea137723a6355d36d Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 16:26:11 +0100 Subject: [PATCH 154/158] fix node setup --- .github/workflows/lint-code.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index b348496c3c..168fac653c 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install editorconfig-checker run: npm install -g editorconfig-checker @@ -33,6 +35,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier From 7f6bc73b7577547c28bc6041a8a0d11b4bcb651d Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 16:37:17 +0100 Subject: [PATCH 155/158] fix apt command --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4f370bc280..c6de90d933 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -78,7 +78,7 @@ jobs: if: ${{ needs.setup.outputs.runner == 'ubuntu-20.04' && needs.setup.outputs.python-version == '3.8' }} run: | sudo apt update - sudo apt remove git git-man + sudo apt remove -y git git-man sudo add-apt-repository --remove ppa:git-core/ppa sudo apt install -y git - name: Get current date From a84d064d2ade1412d8704d72879692d02d3a7dde Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 17:16:49 +0100 Subject: [PATCH 156/158] reset aws runner for nf-test related things, try it with other workflows --- .github/workflows/create-lint-wf.yml | 4 +++- .github/workflows/create-test-lint-wf-template.yml | 4 +++- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/fix-linting.yml | 2 ++ .github/workflows/push_dockerhub_dev.yml | 2 +- .github/workflows/pytest.yml | 2 +- .github/workflows/tools-api-docs-dev.yml | 2 +- 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 51daec21b7..a3941f3ce1 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -17,7 +17,7 @@ env: jobs: MakeTestWorkflow: - runs-on: ubuntu-latest + runs-on: self-hosted env: NXF_ANSI_LOG: false strategy: @@ -49,6 +49,8 @@ jobs: # Install the Prettier linting tools - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index f1c6c4d149..b14a0fac52 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -20,7 +20,7 @@ env: jobs: RunTestWorkflow: - runs-on: ubuntu-latest + runs-on: self-hosted env: NXF_ANSI_LOG: false strategy: @@ -53,6 +53,8 @@ jobs: # Install the Prettier linting tools - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 4f66adae6b..7b6a868f9e 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -17,7 +17,7 @@ env: jobs: RunTestWorkflow: - runs-on: ubuntu-latest + runs-on: self-hosted env: NXF_ANSI_LOG: false strategy: diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index a6cf149569..d846151205 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -25,6 +25,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - uses: actions/setup-node@v4 + with: + node-version: "20" - name: Install Prettier run: npm install -g prettier @prettier/plugin-php diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index 169a917d83..1230bfc9d3 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -13,7 +13,7 @@ concurrency: jobs: push_dockerhub: name: Push new Docker image to Docker Hub (dev) - runs-on: ubuntu-latest + runs-on: self-hosted # Only run for the nf-core repo, for releases and merged PRs if: ${{ github.repository == 'nf-core/tools' }} env: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c6de90d933..0561be25e5 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -58,7 +58,7 @@ jobs: name: Test with Python ${{ needs.setup.outputs.python-version }} on ${{ needs.setup.outputs.runner }} needs: setup if: ${{ needs.setup.outputs.run-tests == 'true' }} - runs-on: ["self-hosted"] + runs-on: ${{ needs.setup.outputs.runner }} steps: - uses: actions/checkout@v2 name: Check out source-code repository diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index 7de8913e03..f6106bd8b5 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -20,7 +20,7 @@ concurrency: jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Check out source-code repository From 9599287f7fe67fab93e57dee4a4dc67c8a91eec0 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 17:24:01 +0100 Subject: [PATCH 157/158] try fixing aws runs --- nf_core/pipeline-template/nextflow.config | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 6a6bee6a83..31917a3e31 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -121,6 +121,7 @@ profiles { shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + runOptions = '-u $(id -u):$(id -g)' } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' From 88e8bce12ac8c967e0fc36e153388f6f772e0c47 Mon Sep 17 00:00:00 2001 From: mashehu Date: Wed, 29 Nov 2023 23:18:08 +0100 Subject: [PATCH 158/158] run on github runners --- .github/workflows/create-test-lint-wf-template.yml | 2 +- .github/workflows/create-test-wf.yml | 2 +- .github/workflows/pytest.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index b14a0fac52..4be3098629 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -20,7 +20,7 @@ env: jobs: RunTestWorkflow: - runs-on: self-hosted + runs-on: ubuntu-latest env: NXF_ANSI_LOG: false strategy: diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 7b6a868f9e..4f66adae6b 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -17,7 +17,7 @@ env: jobs: RunTestWorkflow: - runs-on: self-hosted + runs-on: ubuntu-latest env: NXF_ANSI_LOG: false strategy: diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0561be25e5..9f8172a3be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -25,7 +25,7 @@ env: jobs: setup: - runs-on: ["self-hosted"] + runs-on: ["ubuntu-latest"] strategy: matrix: python-version: ["3.8", "3.12"]