diff --git a/CHANGELOG.md b/CHANGELOG.md index 5321db1f5e..5aebc3367e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ ### Modules * Added consistency checks between installed modules and `modules.json` ([#1199](https://github.com/nf-core/tools/issues/1199)) +* Added support excluding or specifying version of modules in `.nf-core.yml` when updating with `nf-core modules install --all` ([#1204](https://github.com/nf-core/tools/issues/1204)) * Created `nf-core modules update` and removed updating options from `nf-core modules install` * Added missing function call to `nf-core lint` ([#1198](https://github.com/nf-core/tools/issues/1198)) * Fix `nf-core lint` not filtering modules test when run with `--key` ([#1203](https://github.com/nf-core/tools/issues/1203)) diff --git a/README.md b/README.md index 4b5ded3202..b40803c983 100644 --- a/README.md +++ b/README.md @@ -990,13 +990,12 @@ INFO Installing cat/fastq INFO Downloaded 3 files to ./modules/nf-core/modules/cat/fastq ``` -You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`. +You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir `. -There are four flags that you can use with this command: +There are three additional flags that you can use when installing a module: -* `--dir `: Specify a pipeline directory other than the current working directory. -* `--prompt`: Select the module version using a cli prompt. * `--force`: Overwrite a previously installed version of the module. +* `--prompt`: Select the module version using a cli prompt. * `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. ### Update modules in a pipeline @@ -1018,16 +1017,47 @@ INFO Updating 'nf-core/modules/fastqc' INFO Downloaded 3 files to ./modules/nf-core/modules/fastqc ``` -You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`. +You can pass the module name as an optional argument to `nf-core modules update` instead of using the cli prompt, eg: `nf-core modules update fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir `. -There are five flags that you can use with this command: +There are four additional flags that you can use with this command: -* `--dir `: Specify a pipeline directory other than the current working directory. * `--force`: Reinstall module even if it appears to be up to date * `--prompt`: Select the module version using a cli prompt. * `--sha `: Install the module at a specific commit from the `nf-core/modules` repository. * `--all`: Use this flag to run the command on all modules in the pipeline. +If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file: + +```yaml +update: + nf-core/modules: + star/align: False +``` + +If you want this module to be updated only to a specific version (or downgraded), you could instead specifiy the version: + +```yaml +update: + nf-core/modules: + star/align: "e937c7950af70930d1f34bb961403d9d2aa81c7" +``` + +This also works at the repository level. For example, if you want to exclude all modules installed from `nf-core/modules` from being updated you could add: + +```yaml +update: + nf-core/modules: False +``` + +or if you want all modules in `nf-core/modules` at a specific version: + +```yaml +update: + nf-core/modules: "e937c7950af70930d1f34bb961403d9d2aa81c7" +``` + +Note that the module versions specified in the `.nf-core.yml` file has higher precedence than versions specified with the command line flags, thus aiding you in writing reproducible pipelines. + ### Remove a module from a pipeline To delete a module from your pipeline, run `nf-core modules remove`. diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index aed84cf881..9e642c06b7 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -54,80 +54,72 @@ def install(self, module): log.error("Module '{}' not found in list of available modules.".format(module)) log.info("Use the command 'nf-core modules list' to view available software") return False - repos_and_modules = [(self.modules_repo, module)] # Load 'modules.json' modules_json = self.load_modules_json() if not modules_json: return False - exit_value = True - for modules_repo, module in repos_and_modules: - if not module_exist_in_repo(module, modules_repo): - warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})" - log.warning(warn_msg) - exit_value = False - continue + if not module_exist_in_repo(module, self.modules_repo): + warn_msg = f"Module '{module}' not found in remote '{self.modules_repo.name}' ({self.modules_repo.branch})" + log.warning(warn_msg) + return False - if modules_repo.name in modules_json["repos"]: - current_entry = modules_json["repos"][modules_repo.name].get(module) - else: - current_entry = None + if self.modules_repo.name in modules_json["repos"]: + current_entry = modules_json["repos"][self.modules_repo.name].get(module) + else: + current_entry = None - # Set the install folder based on the repository name - install_folder = [modules_repo.owner, modules_repo.repo] + # Set the install folder based on the repository name + install_folder = [self.modules_repo.owner, self.modules_repo.repo] - # Compute the module directory - module_dir = os.path.join(self.dir, "modules", *install_folder, module) + # Compute the module directory + module_dir = os.path.join(self.dir, "modules", *install_folder, module) - # Check that the module is not already installed - if (current_entry is not None and os.path.exists(module_dir)) and not self.force: + # Check that the module is not already installed + if (current_entry is not None and os.path.exists(module_dir)) and not self.force: - log.error(f"Module is already installed.") - log.info( - f"To update '{module}' run 'nf-core modules update {module}'. To force reinstallation use '--force'" - ) - exit_value = False - continue - - if self.sha: - version = self.sha - elif self.prompt: - try: - version = nf_core.modules.module_utils.prompt_module_version_sha( - module, - installed_sha=current_entry["git_sha"] if not current_entry is None else None, - modules_repo=modules_repo, - ) - except SystemError as e: - log.error(e) - exit_value = False - continue - else: - # Fetch the latest commit for the module - try: - git_log = get_module_git_log(module, modules_repo=modules_repo, per_page=1, page_nbr=1) - except UserWarning: - log.error(f"Was unable to fetch version of module '{module}'") - exit_value = False - continue - latest_version = git_log[0]["git_sha"] - version = latest_version - - if self.force: - log.info(f"Removing installed version of '{modules_repo.name}/{module}'") - self.clear_module_dir(module, module_dir) - - log.info(f"{'Rei' if self.force else 'I'}nstalling '{modules_repo.name}/{module}'") - log.debug( - f"Installing module '{module}' at modules hash {modules_repo.modules_current_hash} from {self.modules_repo.name}" + log.error(f"Module is already installed.") + repo_flag = "" if self.modules_repo.name == "nf-core/modules" else f"-g {self.modules_repo.name} " + branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} " + + log.info( + f"To update '{module}' run 'nf-core modules {repo_flag}{branch_flag}update {module}'. To force reinstallation use '--force'" ) + return False - # Download module files - if not self.download_module_file(module, version, modules_repo, install_folder, module_dir): - exit_value = False - continue + if self.sha: + version = self.sha + elif self.prompt: + try: + version = nf_core.modules.module_utils.prompt_module_version_sha( + module, + installed_sha=current_entry["git_sha"] if not current_entry is None else None, + modules_repo=self.modules_repo, + ) + except SystemError as e: + log.error(e) + return False + else: + # Fetch the latest commit for the module + try: + git_log = get_module_git_log(module, modules_repo=self.modules_repo, per_page=1, page_nbr=1) + except UserWarning: + log.error(f"Was unable to fetch version of module '{module}'") + return False + version = git_log[0]["git_sha"] + + if self.force: + log.info(f"Removing installed version of '{self.modules_repo.name}/{module}'") + self.clear_module_dir(module, module_dir) + + log.info(f"{'Rei' if self.force else 'I'}nstalling '{self.modules_repo.name}/{module}'") + log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.name}") + + # Download module files + if not self.download_module_file(module, version, self.modules_repo, install_folder, module_dir): + return False - # Update module.json with newly installed module - self.update_modules_json(modules_json, modules_repo.name, module, version) - return exit_value + # Update module.json with newly installed module + self.update_modules_json(modules_json, self.modules_repo.name, module, version) + return True diff --git a/nf_core/modules/modules_repo.py b/nf_core/modules/modules_repo.py index 635bb7bcb6..d6d2871ecd 100644 --- a/nf_core/modules/modules_repo.py +++ b/nf_core/modules/modules_repo.py @@ -29,7 +29,6 @@ def __init__(self, repo="nf-core/modules", branch="master"): self.owner, self.repo = self.name.split("/") self.modules_file_tree = {} - self.modules_current_hash = None self.modules_avail_module_names = [] def verify_modules_repo(self): @@ -65,7 +64,6 @@ def get_modules_file_tree(self): Fetch the file list from the repo, using the GitHub API Sets self.modules_file_tree - self.modules_current_hash self.modules_avail_module_names """ api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch) @@ -80,7 +78,6 @@ def get_modules_file_tree(self): result = r.json() assert result["truncated"] == False - self.modules_current_hash = result["sha"] self.modules_file_tree = result["tree"] for f in result["tree"]: if f["path"].startswith(f"modules/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]: diff --git a/nf_core/modules/update.py b/nf_core/modules/update.py index b8dc2132ae..c60a90c03e 100644 --- a/nf_core/modules/update.py +++ b/nf_core/modules/update.py @@ -31,6 +31,9 @@ def update(self, module): # Verify that 'modules.json' is consistent with the installed modules self.modules_json_up_to_date() + tool_config = nf_core.utils.load_tools_config() + update_config = tool_config.get("update", {}) + if not self.update_all: # Get the available modules try: @@ -55,25 +58,86 @@ def update(self, module): style=nf_core.utils.nfcore_question_style, ).unsafe_ask() + sha = self.sha + if module in update_config.get(self.modules_repo.name, {}): + config_entry = update_config[self.modules_repo.name].get(module) + if config_entry is not None and config_entry is not True: + if config_entry is False: + log.error("Module's update entry in '.nf-core.yml' is set to False") + return False + elif isinstance(config_entry, str): + if self.sha: + log.warning( + "Found entry in '.nf-core.yml' for module " + "which will override version specified with '--sha'" + ) + sha = config_entry + else: + log.error("Module's update entry in '.nf-core.yml' is of wrong type") + return False + # Check that the supplied name is an available module if module and module not in self.modules_repo.modules_avail_module_names: log.error("Module '{}' not found in list of available modules.".format(module)) log.info("Use the command 'nf-core modules list remote' to view available software") return False - repos_and_modules = [(self.modules_repo, module)] + repos_mods_shas = [(self.modules_repo, module, sha)] + else: if module: raise UserWarning("You cannot specify a module and use the '--all' flag at the same time") self.get_pipeline_modules() - repos_and_modules = [ - (ModulesRepo(repo=repo_name), modules) for repo_name, modules in self.module_names.items() + + # Filter out modules that should not be updated or assign versions if there are any + skipped_repos = [] + skipped_modules = [] + repos_mods_shas = {} + for repo_name, modules in self.module_names.items(): + if repo_name not in update_config or update_config[repo_name] is True: + repos_mods_shas[repo_name] = [] + for module in modules: + repos_mods_shas[repo_name].append((module, self.sha)) + elif isinstance(update_config[repo_name], dict): + repo_config = update_config[repo_name] + repos_mods_shas[repo_name] = [] + for module in modules: + if module not in repo_config or repo_config[module] is True: + repos_mods_shas[repo_name].append((module, self.sha)) + elif isinstance(repo_config[module], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = repo_config[module] + repos_mods_shas[repo_name].append((module, custom_sha)) + else: + # Otherwise the entry must be 'False' and we should ignore the module + skipped_modules.append(f"{repo_name}/{module}") + elif isinstance(update_config[repo_name], str): + # If a string is given it is the commit SHA to which we should update to + custom_sha = update_config[repo_name] + repos_mods_shas[repo_name] = [] + for module in modules: + repos_mods_shas[repo_name].append((module, custom_sha)) + else: + skipped_repos.append(repo_name) + + if skipped_repos: + skipped_str = "', '".join(skipped_repos) + log.info(f"Skipping modules in repositor{'y' if len(skipped_repos) == 1 else 'ies'}: '{skipped_str}'") + + if skipped_modules: + skipped_str = "', '".join(skipped_modules) + log.info(f"Skipping module{'' if len(skipped_modules) == 1 else 's'}: '{skipped_str}'") + + repos_mods_shas = [ + (ModulesRepo(repo=repo_name), mods_shas) for repo_name, mods_shas in repos_mods_shas.items() ] - # Load the modules file trees - for repo, _ in repos_and_modules: + + for repo, _ in repos_mods_shas: repo.get_modules_file_tree() - repos_and_modules = [(repo, module) for repo, modules in repos_and_modules for module in modules] + + # Flatten the list + repos_mods_shas = [(repo, mod, sha) for repo, mods_shas in repos_mods_shas for mod, sha in mods_shas] # Load 'modules.json' modules_json = self.load_modules_json() @@ -81,7 +145,7 @@ def update(self, module): return False exit_value = True - for modules_repo, module in repos_and_modules: + for modules_repo, module, sha in repos_mods_shas: if not module_exist_in_repo(module, modules_repo): warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})" if self.update_all: @@ -101,8 +165,8 @@ def update(self, module): # Compute the module directory module_dir = os.path.join(self.dir, "modules", *install_folder, module) - if self.sha: - version = self.sha + if sha: + version = sha elif self.prompt: try: version = nf_core.modules.module_utils.prompt_module_version_sha( @@ -135,9 +199,7 @@ def update(self, module): continue log.info(f"Updating '{modules_repo.name}/{module}'") - log.debug( - f"Updating module '{module}' to {modules_repo.modules_current_hash} from {self.modules_repo.name}" - ) + log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}") log.debug(f"Removing old version of module '{module}'") self.clear_module_dir(module, module_dir) diff --git a/nf_core/utils.py b/nf_core/utils.py index c5d61523d7..de120f10d7 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -771,7 +771,9 @@ def load_tools_config(dir="."): except FileNotFoundError: log.debug(f"No tools config file found: {config_fn}") return {} - + if tools_config is None: + # If the file is empty + return {} return tools_config