From d445eeb51d6f0ac95183fc3aef6147820d638d5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 5 Aug 2022 17:04:18 +0200 Subject: [PATCH 01/21] Give better suggestion when modules.json file is out-of-date. --- nf_core/modules/list.py | 2 +- nf_core/modules/modules_json.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index 7ae3e08962..816ce33e93 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -93,7 +93,7 @@ def pattern_msg(keywords): repo_modules = repo_entry.get("modules") if repo_modules is None: raise UserWarning( - "You 'modules.json' file is not up to date. Please remove it and rerun the command" + "You 'modules.json' file is not up to date. You can fix it by running 'nf-core modules update'." ) module_entry = repo_modules.get(module) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 99f1d38194..8451949569 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -284,7 +284,7 @@ def check_up_to_date(self): if "git_url" not in values or "base_path" not in values: raise UserWarning( "The 'modules.json' file is not up to date. " - "Please reinstall it by removing it and rerunning the command." + "You can fix it by running 'nf-core modules update'." ) # If there are any modules left in 'modules.json' after all installed are removed, # we try to reinstall them From 1f1ff4fdb66165c12bc5dd577dea80c6517bc258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 5 Aug 2022 17:08:06 +0200 Subject: [PATCH 02/21] change message in new commands --- nf_core/modules/modules_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f50365441b..24ea90897a 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -354,7 +354,7 @@ def unsynced_modules(self): if "git_sha" not in modules[module] or "branch" not in modules[module]: raise UserWarning( "The 'modules.json' file is not up to date. " - "Please reinstall it by removing it and rerunning the command." + "You can fix it by running 'nf-core modules update'." ) missing_installation[module_repo_name]["modules"].pop(module) if len(missing_installation[module_repo_name]["modules"]) == 0: @@ -434,7 +434,7 @@ def check_up_to_date(self): if not self.has_git_url_and_base_path(): raise UserWarning( "The 'modules.json' file is not up to date. " - "Please reinstall it by removing it and rerunning the command." + "You can fix it by running 'nf-core modules update'." ) missing_from_modules_json, missing_installation = self.unsynced_modules() From f7dbfe41caad58fd569a3f09821dc5851d2e6330 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 5 Aug 2022 15:24:57 +0000 Subject: [PATCH 03/21] [automated] Fix linting with Prettier --- nf_core/modules/modules_json.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 24ea90897a..aab8027af1 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -433,8 +433,7 @@ def check_up_to_date(self): self.load() if not self.has_git_url_and_base_path(): raise UserWarning( - "The 'modules.json' file is not up to date. " - "You can fix it by running 'nf-core modules update'." + "The 'modules.json' file is not up to date. " "You can fix it by running 'nf-core modules update'." ) missing_from_modules_json, missing_installation = self.unsynced_modules() From a7a7b295302a37779687806143a5adc64ee09e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 5 Aug 2022 18:12:45 +0200 Subject: [PATCH 04/21] recreate modules.json instead of complaining --- nf_core/modules/modules_json.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 24ea90897a..d25a32eea2 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -432,10 +432,9 @@ def check_up_to_date(self): """ self.load() if not self.has_git_url_and_base_path(): - raise UserWarning( - "The 'modules.json' file is not up to date. " - "You can fix it by running 'nf-core modules update'." - ) + log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") + self.create() + self.load() missing_from_modules_json, missing_installation = self.unsynced_modules() From d600c8f33244ad87a2ea2252ae15b3a55af28086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Fri, 5 Aug 2022 18:28:15 +0200 Subject: [PATCH 05/21] remove `has_module_file` method, as all commands relying on a module.json file run also `check_up_to_date`, which now rebuilds outdates files --- nf_core/modules/modules_command.py | 8 -------- nf_core/modules/modules_json.py | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 9f82c5bfa5..4aad35978c 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -62,16 +62,8 @@ def has_valid_directory(self): nf_config = os.path.join(self.dir, "nextflow.config") if not os.path.exists(main_nf) and not os.path.exists(nf_config): raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") - self.has_modules_file() return True - def has_modules_file(self): - """Checks whether a module.json file has been created and creates one if it is missing""" - modules_json_path = os.path.join(self.dir, "modules.json") - if not os.path.exists(modules_json_path): - log.info("Creating missing 'module.json' file.") - ModulesJson(self.dir).create() - def clear_module_dir(self, module_name, module_dir): """Removes all files in the module directory""" try: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index d25a32eea2..ea2d565901 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -368,7 +368,7 @@ def unsynced_modules(self): def has_git_url_and_base_path(self): """ Check that that all repo entries in the modules.json - has a git url and a base_path + have a git url and a base_path Returns: (bool): True if they are found for all repos, False otherwise From dc2408c8809a1a415356da9b35adef7a2d382be2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 8 Aug 2022 11:26:38 +0200 Subject: [PATCH 06/21] add `has_modules_file()` back in for lint and remove command --- nf_core/lint/__init__.py | 3 ++- nf_core/modules/modules_command.py | 7 +++++++ nf_core/modules/remove.py | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index 229bc79e59..d61b7ba48c 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -86,8 +86,9 @@ def run_linting( # Create the modules lint object module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir) - # Verify that the pipeline is correctly configured + # Verify that the pipeline is correctly configured and has a modules.json file module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() # Run only the tests we want if key: diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 4aad35978c..7a5f13337e 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -64,6 +64,13 @@ def has_valid_directory(self): raise UserWarning(f"Could not find a 'main.nf' or 'nextflow.config' file in '{self.dir}'") return True + def has_modules_file(self): + """Checks whether a module.json file has been created and creates one if it is missing""" + modules_json_path = os.path.join(self.dir, "modules.json") + if not os.path.exists(modules_json_path): + log.info("Creating missing 'module.json' file.") + ModulesJson(self.dir).create() + def clear_module_dir(self, module_name, module_dir): """Removes all files in the module directory""" try: diff --git a/nf_core/modules/remove.py b/nf_core/modules/remove.py index 4b14745a92..59e8e4822d 100644 --- a/nf_core/modules/remove.py +++ b/nf_core/modules/remove.py @@ -28,8 +28,9 @@ def remove(self, module): log.error("You cannot remove a module in a clone of nf-core/modules") return False - # Check whether pipelines is valid + # Check whether pipeline is valid and with a modules.json file self.has_valid_directory() + self.has_modules_file() repo_name = self.modules_repo.fullname if module is None: From 6dd5b8b3ebbb2aabef21165b2fee78804fd086e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 8 Aug 2022 15:51:16 +0200 Subject: [PATCH 07/21] Improve error handling, assign `modules_json` variable already in `.create()` --- nf_core/modules/modules_json.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index ea2d565901..6801533979 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -74,11 +74,12 @@ def create(self): modules_json["repos"][repo_name]["modules"] = self.determine_module_branches_and_shas( repo_name, remote_url, base_path, module_names ) - + # write the modules.json file and assign it to the object modules_json_path = Path(self.dir, "modules.json") with open(modules_json_path, "w") as fh: json.dump(modules_json, fh, indent=4) fh.write("\n") + self.modules_json = modules_json def get_pipeline_module_repositories(self, modules_dir, repos=None): """ @@ -430,11 +431,13 @@ def check_up_to_date(self): If a module is installed but the entry in 'modules.json' is missing we iterate through the commit log in the remote to try to determine the SHA. """ - self.load() - if not self.has_git_url_and_base_path(): + try: + self.load() + if not self.has_git_url_and_base_path(): + raise KeyError + except (UserWarning, KeyError): log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") self.create() - self.load() missing_from_modules_json, missing_installation = self.unsynced_modules() From fc7693e12b41aae052ac94b321b85bd8dbf15254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Mon, 8 Aug 2022 16:44:16 +0200 Subject: [PATCH 08/21] add test for module.json with incorrect format --- tests/modules/modules_json.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 35808c529b..054acdf5ed 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -176,3 +176,16 @@ def test_mod_json_dump(self): with open(mod_json_path, "r") as f: mod_json_new = json.load(f) assert mod_json == mod_json_new + + +def test_mod_json_with_missing_base_path_fail(self): + # Load module.json and remove the base_path entry + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_obj["base_path"].pop("base_path") + # write the new module.json and load it + mod_json_obj.dump() + mod_json_path = os.path.join(self.pipeline_dir, "modules.json") + with open(mod_json_path, "r") as f: + mod_json_broken = json.load(f) + mod_json_broken.check_up_to_date() + assert mod_json == mod_json_broken From ca39fb923b7889afc4bb604ee71072265e84d4dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 9 Aug 2022 14:17:19 +0200 Subject: [PATCH 09/21] fix test for incorrect mod_json --- tests/modules/modules_json.py | 17 ++++++++++------- tests/test_modules.py | 2 ++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 054acdf5ed..4706fcfecc 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -1,6 +1,7 @@ import json import os import shutil +import copy from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ( @@ -181,11 +182,13 @@ def test_mod_json_dump(self): def test_mod_json_with_missing_base_path_fail(self): # Load module.json and remove the base_path entry mod_json_obj = ModulesJson(self.pipeline_dir) - mod_json_obj["base_path"].pop("base_path") - # write the new module.json and load it + mod_json_orig = mod_json_obj.get_modules_json() + mod_json = copy.deepcopy(mod_json_orig) + mod_json["repos"]["nf-core/modules"].pop("base_path") + # save the altered module.json and load it again to check if it will fix itself + mod_json_obj.modules_json = mod_json mod_json_obj.dump() - mod_json_path = os.path.join(self.pipeline_dir, "modules.json") - with open(mod_json_path, "r") as f: - mod_json_broken = json.load(f) - mod_json_broken.check_up_to_date() - assert mod_json == mod_json_broken + mod_json_obj_new = ModulesJson(self.pipeline_dir) + mod_json_obj_new.check_up_to_date() + mod_json_new = mod_json_obj_new.get_modules_json() + assert mod_json_orig == mod_json_new diff --git a/tests/test_modules.py b/tests/test_modules.py index 2ceb2ab08f..6df7ccfcf6 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -142,7 +142,9 @@ def test_modulesrepo_class(self): test_mod_json_up_to_date_module_removed, test_mod_json_up_to_date_reinstall_fails, test_mod_json_update, + test_mod_json_with_missing_base_path_fail, ) + from .modules.patch import ( test_create_patch_change, test_create_patch_no_change, From 6d3717a8b707657643148fe6054a0edd23c4c357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 9 Aug 2022 14:21:41 +0200 Subject: [PATCH 10/21] fix isort --- tests/modules/modules_json.py | 2 +- tests/test_modules.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index 4706fcfecc..c191cff51c 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -1,7 +1,7 @@ +import copy import json import os import shutil -import copy from nf_core.modules.modules_json import ModulesJson from nf_core.modules.modules_repo import ( diff --git a/tests/test_modules.py b/tests/test_modules.py index 6df7ccfcf6..f705a54f29 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -144,7 +144,6 @@ def test_modulesrepo_class(self): test_mod_json_update, test_mod_json_with_missing_base_path_fail, ) - from .modules.patch import ( test_create_patch_change, test_create_patch_no_change, From 47d551bf7fbc07df14cfdd804e704cc264452668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 9 Aug 2022 14:31:34 +0200 Subject: [PATCH 11/21] updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 36302571f5..1a6dd14543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ - Fix misc. issues with `--branch` and `--base-path` ([#1726](https://github.com/nf-core/tools/issues/1726)) - Add `branch` field to module entries in `modules.json` to record what branch a module was installed from ([#1728](https://github.com/nf-core/tools/issues/1728)) - Fix unbound variable issues and minor refactoring [#1742](https://github.com/nf-core/tools/pull/1742/) +- Recreate modules.json file instead of complaining about incorrectly formatted file. ([#1741](https://github.com/nf-core/tools/pull/1741) ## [v2.4.1 - Cobolt Koala Patch](https://github.com/nf-core/tools/releases/tag/2.4) - [2022-05-16] From 5c05171153f3262b9fbc6fc94a99dbf344923d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 16 Aug 2022 17:52:59 +0200 Subject: [PATCH 12/21] rename `has_git_url_and_base_path` to `has_correct_format` and include check for missing `git_sha` and `branch` fields in it --- nf_core/modules/modules_json.py | 35 ++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 6801533979..2ac8808b7e 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -249,7 +249,7 @@ def determine_module_branches_and_shas(self, repo_name, remote_url, base_path, m {"name": "Remove the files", "value": 1}, ], style=nf_core.utils.nfcore_question_style, - ) + ).unsafe_ask() if action == 0: sb_local.append(module) else: @@ -324,10 +324,9 @@ def unsynced_modules(self): directories containing a 'main.nf' file Returns: - (untrack_dirs ([ Path ]), - missing_installation (dict)): Directories that are not tracked - by the modules.json file, and modules in the modules.json that - where the installation directory is missing + (untrack_dirs ([ Path ]), missing_installation (dict)): Directories that are not tracked + by the modules.json file, and modules in the modules.json where + the installation directory is missing """ missing_installation = copy.deepcopy(self.modules_json["repos"]) dirs = [ @@ -346,19 +345,19 @@ def unsynced_modules(self): if module_repo_name is not None: # If it does, check if the module is in the 'modules.json' file module = str(dir.relative_to(module_repo_name)) + module_repo = missing_installation[module_repo_name] - if module not in missing_installation[module_repo_name].get("modules", {}): + if module not in module_repo.get("modules", {}): untracked_dirs.append(dir) else: # Check if the entry has a git sha and branch before removing - modules = missing_installation[module_repo_name]["modules"] + modules = module_repo["modules"] if "git_sha" not in modules[module] or "branch" not in modules[module]: - raise UserWarning( - "The 'modules.json' file is not up to date. " - "You can fix it by running 'nf-core modules update'." + self.determine_module_branches_and_shas( + module, module_repo["git_url"], module_repo["base_path"], [module] ) - missing_installation[module_repo_name]["modules"].pop(module) - if len(missing_installation[module_repo_name]["modules"]) == 0: + module_repo["modules"].pop(module) + if len(module_repo["modules"]) == 0: missing_installation.pop(module_repo_name) else: # If it is not, add it to the list of missing modules @@ -366,7 +365,7 @@ def unsynced_modules(self): return untracked_dirs, missing_installation - def has_git_url_and_base_path(self): + def has_correct_format(self): """ Check that that all repo entries in the modules.json have a git url and a base_path @@ -376,7 +375,12 @@ def has_git_url_and_base_path(self): """ for repo_entry in self.modules_json.get("repos", {}).values(): if "git_url" not in repo_entry or "base_path" not in repo_entry: - return False + raise KeyError + for repo_name, modules in repo_entry.get("modules", {}).items(): + if "git_sha" not in modules or "branch" not in modules: + self.determine_module_branches_and_shas( + repo_name, repo_entry["git_url"], repo_entry["base_path"], [repo_name] + ) return True def reinstall_repo(self, repo_name, remote_url, base_path, module_entries): @@ -433,8 +437,7 @@ def check_up_to_date(self): """ try: self.load() - if not self.has_git_url_and_base_path(): - raise KeyError + self.has_correct_format() except (UserWarning, KeyError): log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") self.create() From 3f209ab15009253970c105b6e15091f6f5cf2170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 18 Aug 2022 17:54:49 +0200 Subject: [PATCH 13/21] rename to has_git_url_and_modules (in expectation of removal of base_path) and return booleans instead of warnings --- nf_core/modules/list.py | 4 ---- nf_core/modules/modules_json.py | 34 ++++++++++++++++++++------------- tests/modules/modules_json.py | 32 ++++++++++++++++++++++++++++++- tests/test_modules.py | 5 ++++- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/nf_core/modules/list.py b/nf_core/modules/list.py index d17648ef25..c093888c1c 100644 --- a/nf_core/modules/list.py +++ b/nf_core/modules/list.py @@ -91,10 +91,6 @@ def pattern_msg(keywords): repo_entry = modules_json["repos"].get(repo_name, {}) for module in sorted(modules): repo_modules = repo_entry.get("modules") - if repo_modules is None: - raise UserWarning( - "You 'modules.json' file is not up to date. You can fix it by running 'nf-core modules update'." - ) module_entry = repo_modules.get(module) if module_entry: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 5cdb8420a7..0ec3a8044c 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -366,22 +366,29 @@ def unsynced_modules(self): return untracked_dirs, missing_installation - def has_correct_format(self): + def has_git_url_and_modules(self): """ - Check that that all repo entries in the modules.json - have a git url and a base_path - + Check that all repo entries in the modules.json + has a git url and a base_path with non-empty string values and a modules dict entry Returns: (bool): True if they are found for all repos, False otherwise """ for repo_entry in self.modules_json.get("repos", {}).values(): - if "git_url" not in repo_entry or "base_path" not in repo_entry: - raise KeyError - for repo_name, modules in repo_entry.get("modules", {}).items(): - if "git_sha" not in modules or "branch" not in modules: - self.determine_module_branches_and_shas( - repo_name, repo_entry["git_url"], repo_entry["base_path"], [repo_name] - ) + if "git_url" not in repo_entry or "base_path" not in repo_entry or "modules" not in repo_entry: + log.warning(f"modules.json entry {repo_entry} does not have a git_url, base_path or modules entry") + return False + elif ( + type(repo_entry["git_url"]) != str + or repo_entry["git_url"] == "" + or type(repo_entry["base_path"]) != str + or repo_entry["base_path"] == "" + or type(repo_entry["modules"]) != dict + or repo_entry["modules"] == {} + ): + log.warning( + f"modules.json entry {repo_entry} has non-string or empty entries for git_url or base_path or modules" + ) + return False return True def reinstall_repo(self, repo_name, remote_url, base_path, module_entries): @@ -438,8 +445,9 @@ def check_up_to_date(self): """ try: self.load() - self.has_correct_format() - except (UserWarning, KeyError): + if not self.has_git_url_and_modules(): + raise UserWarning + except (UserWarning): log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") self.create() diff --git a/tests/modules/modules_json.py b/tests/modules/modules_json.py index c191cff51c..6359142cee 100644 --- a/tests/modules/modules_json.py +++ b/tests/modules/modules_json.py @@ -179,7 +179,7 @@ def test_mod_json_dump(self): assert mod_json == mod_json_new -def test_mod_json_with_missing_base_path_fail(self): +def test_mod_json_with_missing_base_path_entry(self): # Load module.json and remove the base_path entry mod_json_obj = ModulesJson(self.pipeline_dir) mod_json_orig = mod_json_obj.get_modules_json() @@ -192,3 +192,33 @@ def test_mod_json_with_missing_base_path_fail(self): mod_json_obj_new.check_up_to_date() mod_json_new = mod_json_obj_new.get_modules_json() assert mod_json_orig == mod_json_new + + +def test_mod_json_with_empty_base_path_value(self): + # Load module.json and remove the base_path entry + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_orig = mod_json_obj.get_modules_json() + mod_json = copy.deepcopy(mod_json_orig) + mod_json["repos"]["nf-core/modules"]["base_path"] = "" + # save the altered module.json and load it again to check if it will fix itself + mod_json_obj.modules_json = mod_json + mod_json_obj.dump() + mod_json_obj_new = ModulesJson(self.pipeline_dir) + mod_json_obj_new.check_up_to_date() + mod_json_new = mod_json_obj_new.get_modules_json() + assert mod_json_orig == mod_json_new + + +def test_mod_json_with_missing_modules_entry(self): + # Load module.json and remove the base_path entry + mod_json_obj = ModulesJson(self.pipeline_dir) + mod_json_orig = mod_json_obj.get_modules_json() + mod_json = copy.deepcopy(mod_json_orig) + mod_json["repos"]["nf-core/modules"].pop("modules") + # save the altered module.json and load it again to check if it will fix itself + mod_json_obj.modules_json = mod_json + mod_json_obj.dump() + mod_json_obj_new = ModulesJson(self.pipeline_dir) + mod_json_obj_new.check_up_to_date() + mod_json_new = mod_json_obj_new.get_modules_json() + assert mod_json_orig == mod_json_new diff --git a/tests/test_modules.py b/tests/test_modules.py index f705a54f29..053076cc66 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -142,8 +142,11 @@ def test_modulesrepo_class(self): test_mod_json_up_to_date_module_removed, test_mod_json_up_to_date_reinstall_fails, test_mod_json_update, - test_mod_json_with_missing_base_path_fail, + test_mod_json_with_missing_base_path_entry, + test_mod_json_with_empty_base_path_value, + test_mod_json_with_missing_modules_entry, ) + from .modules.patch import ( test_create_patch_change, test_create_patch_no_change, From 6ca88360f0538a9ca5c330abc47c1228f9197c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 18 Aug 2022 18:03:23 +0200 Subject: [PATCH 14/21] fix isort error --- tests/test_modules.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 053076cc66..84005e459a 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -142,11 +142,10 @@ def test_modulesrepo_class(self): test_mod_json_up_to_date_module_removed, test_mod_json_up_to_date_reinstall_fails, test_mod_json_update, - test_mod_json_with_missing_base_path_entry, test_mod_json_with_empty_base_path_value, + test_mod_json_with_missing_base_path_entry, test_mod_json_with_missing_modules_entry, ) - from .modules.patch import ( test_create_patch_change, test_create_patch_no_change, From c743e8413a737e27e197b668e6fc52660330c7c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Thu, 18 Aug 2022 19:59:35 +0200 Subject: [PATCH 15/21] Apply suggestions from code review Co-authored-by: Fabian Egli --- nf_core/modules/modules_json.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 0ec3a8044c..dd738f0cb0 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -378,11 +378,11 @@ def has_git_url_and_modules(self): log.warning(f"modules.json entry {repo_entry} does not have a git_url, base_path or modules entry") return False elif ( - type(repo_entry["git_url"]) != str + not isinstance(repo_entry["git_url"], str) or repo_entry["git_url"] == "" - or type(repo_entry["base_path"]) != str + or not isinstance(repo_entry["base_path"], str) or repo_entry["base_path"] == "" - or type(repo_entry["modules"]) != dict + or not isinstance(repo_entry["modules"], dict) or repo_entry["modules"] == {} ): log.warning( @@ -447,7 +447,7 @@ def check_up_to_date(self): self.load() if not self.has_git_url_and_modules(): raise UserWarning - except (UserWarning): + except UserWarning: log.info("The 'modules.json' file is not up to date. Recreating the 'module.json' file.") self.create() From b1f450150a16f8ac4bf2039d6af3ed445eee3bc5 Mon Sep 17 00:00:00 2001 From: nf-core_bot Date: Mon, 22 Aug 2022 15:28:45 +0200 Subject: [PATCH 16/21] fix isort --- tests/test_modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 147f52c607..acc05b8a33 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -144,7 +144,6 @@ def test_modulesrepo_class(self): test_mod_json_update, test_mod_json_with_empty_modules_value, test_mod_json_with_missing_modules_entry, - test_mod_json_with_missing_modules_entry, ) from .modules.patch import ( test_create_patch_change, From e2f50c487db2556914d746cc43860b3a05783cd1 Mon Sep 17 00:00:00 2001 From: nf-core_bot Date: Mon, 22 Aug 2022 15:36:33 +0200 Subject: [PATCH 17/21] update ubuntu version for docs GHA --- .github/workflows/tools-api-docs-dev.yml | 2 +- .github/workflows/tools-api-docs-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tools-api-docs-dev.yml b/.github/workflows/tools-api-docs-dev.yml index f2ccabb02d..fa6baab438 100644 --- a/.github/workflows/tools-api-docs-dev.yml +++ b/.github/workflows/tools-api-docs-dev.yml @@ -10,7 +10,7 @@ concurrency: jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 steps: - name: Check out source-code repository diff --git a/.github/workflows/tools-api-docs-release.yml b/.github/workflows/tools-api-docs-release.yml index 17e504067e..4e4a572c7a 100644 --- a/.github/workflows/tools-api-docs-release.yml +++ b/.github/workflows/tools-api-docs-release.yml @@ -11,7 +11,7 @@ concurrency: jobs: api-docs: name: Build & push Sphinx API docs - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: dir: From 863c390129c53fd88030f70b6cfda66a6de3945d Mon Sep 17 00:00:00 2001 From: nf-core_bot Date: Mon, 22 Aug 2022 15:41:52 +0200 Subject: [PATCH 18/21] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c09bfc7912..ab5b1fb353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ - Simplify control flow and don't use equality comparison for `None` and booleans - Replace use of the deprecated `distutils` Version object with that from `packaging` ([#1735](https://github.com/nf-core/tools/pull/1735)) - Add code to cancel CI run if a new run starts ([#1760](https://github.com/nf-core/tools/pull/1760)) +- Update ubuntu version to 22.04 the image used in building the API docs ### Modules From db147b44c59941001c66d35c48d57b1dd81842ac Mon Sep 17 00:00:00 2001 From: nf-core_bot Date: Mon, 22 Aug 2022 15:43:42 +0200 Subject: [PATCH 19/21] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab5b1fb353..5fd6e67efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ - Simplify control flow and don't use equality comparison for `None` and booleans - Replace use of the deprecated `distutils` Version object with that from `packaging` ([#1735](https://github.com/nf-core/tools/pull/1735)) - Add code to cancel CI run if a new run starts ([#1760](https://github.com/nf-core/tools/pull/1760)) -- Update ubuntu version to 22.04 the image used in building the API docs +- Update ubuntu version to 22.04 in the image used in building the API docs ### Modules From 364e24021a8eb67e6c88cbc19a5742945293d22c Mon Sep 17 00:00:00 2001 From: mashehu Date: Tue, 23 Aug 2022 13:31:55 +0200 Subject: [PATCH 20/21] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fd6e67efc..f47077bdf4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ - Simplify control flow and don't use equality comparison for `None` and booleans - Replace use of the deprecated `distutils` Version object with that from `packaging` ([#1735](https://github.com/nf-core/tools/pull/1735)) - Add code to cancel CI run if a new run starts ([#1760](https://github.com/nf-core/tools/pull/1760)) -- Update ubuntu version to 22.04 in the image used in building the API docs +- Update to latest ubuntu version for building the API docs ### Modules From 4d608aef26d35b05198da6f5c7cf89b307d951c6 Mon Sep 17 00:00:00 2001 From: Fabian Egli Date: Tue, 23 Aug 2022 15:12:25 +0200 Subject: [PATCH 21/21] add link to PR --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47077bdf4..af9a486b38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,7 +48,7 @@ - Simplify control flow and don't use equality comparison for `None` and booleans - Replace use of the deprecated `distutils` Version object with that from `packaging` ([#1735](https://github.com/nf-core/tools/pull/1735)) - Add code to cancel CI run if a new run starts ([#1760](https://github.com/nf-core/tools/pull/1760)) -- Update to latest ubuntu version for building the API docs +- CI for the API docs generation now uses the ubuntu-latest base image ([#1762](https://github.com/nf-core/tools/pull/1762)) ### Modules