From dd88892c7fa7ff4d238d3a53e1b6faa3a5e750f4 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 1 Dec 2023 09:55:53 +0100 Subject: [PATCH 1/5] Use Path objects for ComponentCreate and update the structure of components templates --- .prettierignore | 6 +- CHANGELOG.md | 1 + nf_core/components/create.py | 75 ++++++++----------- .../{modules => }/environment.yml | 0 nf_core/module-template/{modules => }/main.nf | 0 .../module-template/{modules => }/meta.yml | 0 .../{modules => }/tests/main.nf.test | 0 .../{modules => }/tests/tags.yml | 2 +- .../{subworkflows => }/main.nf | 0 .../{subworkflows => }/meta.yml | 0 .../{subworkflows => }/tests/main.nf.test | 0 .../{subworkflows => }/tests/tags.yml | 0 12 files changed, 37 insertions(+), 47 deletions(-) rename nf_core/module-template/{modules => }/environment.yml (100%) rename nf_core/module-template/{modules => }/main.nf (100%) rename nf_core/module-template/{modules => }/meta.yml (100%) rename nf_core/module-template/{modules => }/tests/main.nf.test (100%) rename nf_core/module-template/{modules => }/tests/tags.yml (67%) rename nf_core/subworkflow-template/{subworkflows => }/main.nf (100%) rename nf_core/subworkflow-template/{subworkflows => }/meta.yml (100%) rename nf_core/subworkflow-template/{subworkflows => }/tests/main.nf.test (100%) rename nf_core/subworkflow-template/{subworkflows => }/tests/tags.yml (100%) diff --git a/.prettierignore b/.prettierignore index 778fb4fb2c..344cafca6e 100644 --- a/.prettierignore +++ b/.prettierignore @@ -3,6 +3,6 @@ adaptivecard.json slackreport.json docs/api/_build testing -nf_core/module-template/modules/meta.yml -nf_core/module-template/modules/tests/tags.yml -nf_core/subworkflow-template/subworkflows/tests/tags.yml +nf_core/module-template/meta.yml +nf_core/module-template/tests/tags.yml +nf_core/subworkflow-template/tests/tags.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index a68fcf810c..b12d5568f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) - Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)). +- Use Path objects for ComponentCreate and update the structure of components templates. # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] diff --git a/nf_core/components/create.py b/nf_core/components/create.py index b85acdc57f..dad2d2912b 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -7,7 +7,6 @@ import glob import json import logging -import os import re import shutil import subprocess @@ -60,7 +59,7 @@ def __init__( self.bioconda = None self.singularity_container = None self.docker_container = None - self.file_paths: Dict[str, str] = {} + self.file_paths: Dict[str, Path] = {} self.not_empty_template = not empty_template self.migrate_pytest = migrate_pytest @@ -130,11 +129,11 @@ def create(self): # Determine the component name self.component_name = self.component - self.component_dir = self.component + self.component_dir = Path(self.component) if self.subtool: self.component_name = f"{self.component}/{self.subtool}" - self.component_dir = os.path.join(self.component, self.subtool) + self.component_dir = Path(self.component, self.subtool) self.component_name_underscore = self.component_name.replace("/", "_") @@ -143,7 +142,7 @@ def create(self): if self.migrate_pytest: # Rename the component directory to old - component_old = self.component_dir + "_old" + component_old = self.component_dir.parent / (self.component_dir.name + "_old") component_old_path = Path(self.directory, self.component_type, self.org, component_old) Path(self.directory, self.component_type, self.org, self.component_dir).rename(component_old_path) else: @@ -167,7 +166,7 @@ def create(self): shutil.rmtree(component_old_path) self._print_and_delete_pytest_files() - new_files = list(self.file_paths.values()) + new_files = [str(path) for path in self.file_paths.values()] log.info("Created following files:\n " + "\n ".join(new_files)) def _get_bioconda_tool(self): @@ -276,16 +275,16 @@ def _render_template(self): # Write output to the target file log.debug(f"Writing output to: '{dest_fn}'") - os.makedirs(os.path.dirname(dest_fn), exist_ok=True) + dest_fn.parent.mkdir(exist_ok=True) with open(dest_fn, "w") as fh: log.debug(f"Writing output to: '{dest_fn}'") fh.write(rendered_output) # Mirror file permissions - template_stat = os.stat( - os.path.join(os.path.dirname(nf_core.__file__), f"{self.component_type[:-1]}-template", template_fn) - ) - os.chmod(dest_fn, template_stat.st_mode) + template_stat = ( + Path(nf_core.__file__).parent / f"{self.component_type[:-1]}-template" / template_fn + ).stat() + dest_fn.chmod(template_stat.st_mode) def _collect_name_prompt(self): """ @@ -336,17 +335,17 @@ def _get_component_dirs(self): """ file_paths = {} if self.repo_type == "pipeline": - local_component_dir = os.path.join(self.directory, self.component_type, "local") + local_component_dir = Path(self.directory, self.component_type, "local") # Check whether component file already exists - component_file = os.path.join(local_component_dir, f"{self.component_name}.nf") - if os.path.exists(component_file) and not self.force_overwrite: + component_file = local_component_dir / f"{self.component_name}.nf" + if component_file.exists() and not self.force_overwrite: raise UserWarning( f"{self.component_type[:-1].title()} file exists already: '{component_file}'. Use '--force' to overwrite" ) if self.component_type == "modules": # If a subtool, check if there is a module called the base tool name already - if self.subtool and os.path.exists(os.path.join(local_component_dir, f"{self.component}.nf")): + if self.subtool and (local_component_dir / f"{self.component}.nf").exists(): raise UserWarning( f"Module '{self.component}' exists already, cannot make subtool '{self.component_name}'" ) @@ -359,30 +358,28 @@ def _get_component_dirs(self): ) # Set file paths - file_paths[os.path.join(self.component_type, "main.nf")] = component_file + file_paths["main.nf"] = component_file if self.repo_type == "modules": - component_dir = os.path.join(self.directory, self.component_type, self.org, self.component_dir) + component_dir = Path(self.directory, self.component_type, self.org, self.component_dir) # Check if module/subworkflow directories exist already - if os.path.exists(component_dir) and not self.force_overwrite and not self.migrate_pytest: + if component_dir.exists() and not self.force_overwrite and not self.migrate_pytest: raise UserWarning( f"{self.component_type[:-1]} directory exists: '{component_dir}'. Use '--force' to overwrite" ) if self.component_type == "modules": # If a subtool, check if there is a module called the base tool name already - parent_tool_main_nf = os.path.join( - self.directory, self.component_type, self.org, self.component, "main.nf" - ) - if self.subtool and os.path.exists(parent_tool_main_nf) and not self.migrate_pytest: + parent_tool_main_nf = Path(self.directory, self.component_type, self.org, self.component, "main.nf") + if self.subtool and parent_tool_main_nf.exists() and not self.migrate_pytest: raise UserWarning( f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{self.component_name}'" ) # If no subtool, check that there isn't already a tool/subtool tool_glob = glob.glob( - f"{os.path.join(self.directory, self.component_type, self.org, self.component)}/*/main.nf" + f"{Path(self.directory, self.component_type, self.org, self.component)}/*/main.nf" ) if not self.subtool and tool_glob and not self.migrate_pytest: raise UserWarning( @@ -391,18 +388,12 @@ def _get_component_dirs(self): # Set file paths # For modules - can be tool/ or tool/subtool/ so can't do in template directory structure - file_paths[os.path.join(self.component_type, "main.nf")] = os.path.join(component_dir, "main.nf") - file_paths[os.path.join(self.component_type, "meta.yml")] = os.path.join(component_dir, "meta.yml") + file_paths["main.nf"] = component_dir / "main.nf" + file_paths["meta.yml"] = component_dir / "meta.yml" if self.component_type == "modules": - file_paths[os.path.join(self.component_type, "environment.yml")] = os.path.join( - component_dir, "environment.yml" - ) - file_paths[os.path.join(self.component_type, "tests", "tags.yml")] = os.path.join( - component_dir, "tests", "tags.yml" - ) - file_paths[os.path.join(self.component_type, "tests", "main.nf.test")] = os.path.join( - component_dir, "tests", "main.nf.test" - ) + file_paths["environment.yml"] = component_dir / "environment.yml" + file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml" + file_paths["tests/main.nf.test"] = component_dir / "tests" / "main.nf.test" return file_paths @@ -413,8 +404,7 @@ def _get_username(self): # Try to guess the current user if `gh` is installed author_default = None try: - with open(os.devnull, "w") as devnull: - gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=devnull)) + gh_auth_user = json.loads(subprocess.check_output(["gh", "api", "/user"], stderr=subprocess.DEVNULL)) author_default = f"@{gh_auth_user['login']}" except Exception as e: log.debug(f"Could not find GitHub username using 'gh' cli command: [red]{e}") @@ -432,14 +422,12 @@ def _get_username(self): def _copy_old_files(self, component_old_path): """Copy files from old module to new module""" log.debug("Copying original main.nf file") - shutil.copyfile(component_old_path / "main.nf", self.file_paths[self.component_type + "/main.nf"]) + shutil.copyfile(component_old_path / "main.nf", self.file_paths["main.nf"]) log.debug("Copying original meta.yml file") - shutil.copyfile(component_old_path / "meta.yml", self.file_paths[self.component_type + "/meta.yml"]) + shutil.copyfile(component_old_path / "meta.yml", self.file_paths["meta.yml"]) if self.component_type == "modules": log.debug("Copying original environment.yml file") - shutil.copyfile( - component_old_path / "environment.yml", self.file_paths[self.component_type + "/environment.yml"] - ) + shutil.copyfile(component_old_path / "environment.yml", self.file_paths["environment.yml"]) # Create a nextflow.config file if it contains information other than publishDir pytest_dir = Path(self.directory, "tests", self.component_type, self.org, self.component_dir) nextflow_config = pytest_dir / "nextflow.config" @@ -481,8 +469,9 @@ def _print_and_delete_pytest_files(self): modules_yml = Path(self.directory, "tests", "config", "pytest_modules.yml") with open(modules_yml, "r") as fh: yml_file = yaml.safe_load(fh) - yml_key = self.component_dir if self.component_type == "modules" else f"subworkflows/{self.component_dir}" - del yml_file[yml_key] + yml_key = str(self.component_dir) if self.component_type == "modules" else f"subworkflows/{self.component_dir}" + if yml_key in yml_file: + del yml_file[yml_key] with open(modules_yml, "w") as fh: yaml.dump(yml_file, fh) run_prettier_on_file(modules_yml) diff --git a/nf_core/module-template/modules/environment.yml b/nf_core/module-template/environment.yml similarity index 100% rename from nf_core/module-template/modules/environment.yml rename to nf_core/module-template/environment.yml diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/main.nf similarity index 100% rename from nf_core/module-template/modules/main.nf rename to nf_core/module-template/main.nf diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/meta.yml similarity index 100% rename from nf_core/module-template/modules/meta.yml rename to nf_core/module-template/meta.yml diff --git a/nf_core/module-template/modules/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test similarity index 100% rename from nf_core/module-template/modules/tests/main.nf.test rename to nf_core/module-template/tests/main.nf.test diff --git a/nf_core/module-template/modules/tests/tags.yml b/nf_core/module-template/tests/tags.yml similarity index 67% rename from nf_core/module-template/modules/tests/tags.yml rename to nf_core/module-template/tests/tags.yml index e7fac9f5b9..9f5e352ff4 100644 --- a/nf_core/module-template/modules/tests/tags.yml +++ b/nf_core/module-template/tests/tags.yml @@ -1,2 +1,2 @@ -{{ component_dir }}: +{ { component_dir } }: - "modules/{{ org }}/{{ component_dir }}/**" diff --git a/nf_core/subworkflow-template/subworkflows/main.nf b/nf_core/subworkflow-template/main.nf similarity index 100% rename from nf_core/subworkflow-template/subworkflows/main.nf rename to nf_core/subworkflow-template/main.nf diff --git a/nf_core/subworkflow-template/subworkflows/meta.yml b/nf_core/subworkflow-template/meta.yml similarity index 100% rename from nf_core/subworkflow-template/subworkflows/meta.yml rename to nf_core/subworkflow-template/meta.yml diff --git a/nf_core/subworkflow-template/subworkflows/tests/main.nf.test b/nf_core/subworkflow-template/tests/main.nf.test similarity index 100% rename from nf_core/subworkflow-template/subworkflows/tests/main.nf.test rename to nf_core/subworkflow-template/tests/main.nf.test diff --git a/nf_core/subworkflow-template/subworkflows/tests/tags.yml b/nf_core/subworkflow-template/tests/tags.yml similarity index 100% rename from nf_core/subworkflow-template/subworkflows/tests/tags.yml rename to nf_core/subworkflow-template/tests/tags.yml From 61e479c62f5b50fa150d98a8f024e9eda8de2b03 Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Fri, 1 Dec 2023 10:00:09 +0100 Subject: [PATCH 2/5] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b12d5568f4..07196e45b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490)) - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514)) - Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)). -- Use Path objects for ComponentCreate and update the structure of components templates. +- Use Path objects for ComponentCreate and update the structure of components templates ([#2551](https://github.com/nf-core/tools/pull/2551)). # [v2.10 - Nickel Ostrich](https://github.com/nf-core/tools/releases/tag/2.10) + [2023-09-25] From 93a84ada143de614d3a86fc08637ae805a010dc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Fri, 1 Dec 2023 10:04:20 +0100 Subject: [PATCH 3/5] Fix prettier --- nf_core/module-template/tests/tags.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/tests/tags.yml b/nf_core/module-template/tests/tags.yml index 9f5e352ff4..e7fac9f5b9 100644 --- a/nf_core/module-template/tests/tags.yml +++ b/nf_core/module-template/tests/tags.yml @@ -1,2 +1,2 @@ -{ { component_dir } }: +{{ component_dir }}: - "modules/{{ org }}/{{ component_dir }}/**" From e931f2aa6038e0d0d9155ce2d8f3447e9a9854cb Mon Sep 17 00:00:00 2001 From: mirpedrol Date: Mon, 11 Dec 2023 10:44:28 +0100 Subject: [PATCH 4/5] fix path string concatenation --- nf_core/components/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d5edad2dbd..23f4513c9e 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -142,7 +142,7 @@ def create(self): if self.migrate_pytest: # Rename the component directory to old - component_old_dir = self.component_dir + "_old" + component_old_dir = Path(str(self.component_dir) + "_old") component_parent_path = Path(self.directory, self.component_type, self.org) component_old_path = component_parent_path / component_old_dir component_path = component_parent_path / self.component_dir From 2cbe69a8110140f55fc63a5ba3aa6e755cf22e72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Mon, 11 Dec 2023 10:40:49 +0000 Subject: [PATCH 5/5] fix create module parent directories --- nf_core/components/create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index 23f4513c9e..1d57b789ff 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -278,7 +278,7 @@ def _render_template(self): # Write output to the target file log.debug(f"Writing output to: '{dest_fn}'") - dest_fn.parent.mkdir(exist_ok=True) + dest_fn.parent.mkdir(exist_ok=True, parents=True) with open(dest_fn, "w") as fh: log.debug(f"Writing output to: '{dest_fn}'") fh.write(rendered_output)