Add support for .nf-core.yml to nf-core modules update

CHANGELOG.md

@@ -16,6 +16,7 @@
 ### Modules
 
 * Added consistency checks between installed modules and `modules.json` ([#1199](https://github.com/nf-core/tools/issues/1199))
+* Added support excluding or specifying version of modules in `.nf-core.yml` when updating with `nf-core modules install --all` ([#1204](https://github.com/nf-core/tools/issues/1204))
 * Created `nf-core modules update` and removed updating options from `nf-core modules install`
 * Added missing function call to `nf-core lint` ([#1198](https://github.com/nf-core/tools/issues/1198))
 * Fix `nf-core lint` not filtering modules test when run with `--key` ([#1203](https://github.com/nf-core/tools/issues/1203))

README.md

@@ -990,13 +990,12 @@ INFO     Installing cat/fastq
 INFO     Downloaded 3 files to ./modules/nf-core/modules/cat/fastq
 ```
 
-You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`.
+You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir <pipeline dir>`.
 
-There are four flags that you can use with this command:
+There are three additional flags that you can use when installing a module:
 
-* `--dir <pipeline_dir>`: Specify a pipeline directory other than the current working directory.
-* `--prompt`: Select the module version using a cli prompt.
 * `--force`: Overwrite a previously installed version of the module.
+* `--prompt`: Select the module version using a cli prompt.
 * `--sha <commit_sha>`: Install the module at a specific commit from the `nf-core/modules` repository.
 
 ### Update modules in a pipeline
@@ -1018,16 +1017,47 @@ INFO     Updating 'nf-core/modules/fastqc'
 INFO     Downloaded 3 files to ./modules/nf-core/modules/fastqc
 ```
 
-You can pass the module name as an optional argument to `nf-core modules install` instead of using the cli prompt, eg: `nf-core modules install fastqc`.
+You can pass the module name as an optional argument to `nf-core modules update` instead of using the cli prompt, eg: `nf-core modules update fastqc`. You can specify a pipeline directory other than the current working directory by using the `--dir <pipeline dir>`.
 
-There are five flags that you can use with this command:
+There are four additional flags that you can use with this command:
 
-* `--dir <pipeline_dir>`: Specify a pipeline directory other than the current working directory.
 * `--force`: Reinstall module even if it appears to be up to date
 * `--prompt`: Select the module version using a cli prompt.
 * `--sha <commit_sha>`: Install the module at a specific commit from the `nf-core/modules` repository.
 * `--all`: Use this flag to run the command on all modules in the pipeline.
 
+If you don't want to update certain modules or want to update them to specific versions, you can make use of the `.nf-core.yml` configuration file. For example, you can prevent the `star/align` module installed from `nf-core/modules` from being updated by adding the following to the `.nf-core.yml` file:
+
+```yaml
+update:
+  nf-core/modules:
+    star/align: False
+```
+
+If you want this module to be updated only to a specific version (or downgraded), you could instead specifiy the version:
+
+```yaml
+update:
+  nf-core/modules:
+    star/align: "e937c7950af70930d1f34bb961403d9d2aa81c7"
+```
+
+This also works at the repository level. For example, if you want to exclude all modules installed from `nf-core/modules` from being updated you could add:
+
+```yaml
+update:
+  nf-core/modules: False
+```
+
+or if you want all modules in `nf-core/modules` at a specific version:
+
+```yaml
+update:
+  nf-core/modules: "e937c7950af70930d1f34bb961403d9d2aa81c7"
+```
+
+Note that the module versions specified in the `.nf-core.yml` file has higher precedence than versions specified with the command line flags, thus aiding you in writing reproducible pipelines.
+
 ### Remove a module from a pipeline
 
 To delete a module from your pipeline, run `nf-core modules remove`.

nf_core/modules/install.py

@@ -54,80 +54,72 @@ def install(self, module):
             log.error("Module '{}' not found in list of available modules.".format(module))
             log.info("Use the command 'nf-core modules list' to view available software")
             return False
-        repos_and_modules = [(self.modules_repo, module)]
 
         # Load 'modules.json'
         modules_json = self.load_modules_json()
         if not modules_json:
             return False
 
-        exit_value = True
-        for modules_repo, module in repos_and_modules:
-            if not module_exist_in_repo(module, modules_repo):
-                warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})"
-                log.warning(warn_msg)
-                exit_value = False
-                continue
+        if not module_exist_in_repo(module, self.modules_repo):
+            warn_msg = f"Module '{module}' not found in remote '{self.modules_repo.name}' ({self.modules_repo.branch})"
+            log.warning(warn_msg)
+            return False
 
-            if modules_repo.name in modules_json["repos"]:
-                current_entry = modules_json["repos"][modules_repo.name].get(module)
-            else:
-                current_entry = None
+        if self.modules_repo.name in modules_json["repos"]:
+            current_entry = modules_json["repos"][self.modules_repo.name].get(module)
+        else:
+            current_entry = None
 
-            # Set the install folder based on the repository name
-            install_folder = [modules_repo.owner, modules_repo.repo]
+        # Set the install folder based on the repository name
+        install_folder = [self.modules_repo.owner, self.modules_repo.repo]
 
-            # Compute the module directory
-            module_dir = os.path.join(self.dir, "modules", *install_folder, module)
+        # Compute the module directory
+        module_dir = os.path.join(self.dir, "modules", *install_folder, module)
 
-            # Check that the module is not already installed
-            if (current_entry is not None and os.path.exists(module_dir)) and not self.force:
+        # Check that the module is not already installed
+        if (current_entry is not None and os.path.exists(module_dir)) and not self.force:
 
-                log.error(f"Module is already installed.")
-                log.info(
-                    f"To update '{module}' run 'nf-core modules update {module}'. To force reinstallation use '--force'"
-                )
-                exit_value = False
-                continue
-
-            if self.sha:
-                version = self.sha
-            elif self.prompt:
-                try:
-                    version = nf_core.modules.module_utils.prompt_module_version_sha(
-                        module,
-                        installed_sha=current_entry["git_sha"] if not current_entry is None else None,
-                        modules_repo=modules_repo,
-                    )
-                except SystemError as e:
-                    log.error(e)
-                    exit_value = False
-                    continue
-            else:
-                # Fetch the latest commit for the module
-                try:
-                    git_log = get_module_git_log(module, modules_repo=modules_repo, per_page=1, page_nbr=1)
-                except UserWarning:
-                    log.error(f"Was unable to fetch version of module '{module}'")
-                    exit_value = False
-                    continue
-                latest_version = git_log[0]["git_sha"]
-                version = latest_version
-
-            if self.force:
-                log.info(f"Removing installed version of '{modules_repo.name}/{module}'")
-                self.clear_module_dir(module, module_dir)
-
-            log.info(f"{'Rei' if self.force else 'I'}nstalling '{modules_repo.name}/{module}'")
-            log.debug(
-                f"Installing module '{module}' at modules hash {modules_repo.modules_current_hash} from {self.modules_repo.name}"
+            log.error(f"Module is already installed.")
+            repo_flag = "" if self.modules_repo.name == "nf-core/modules" else f"-g {self.modules_repo.name} "
+            branch_flag = "" if self.modules_repo.branch == "master" else f"-b {self.modules_repo.branch} "
+
+            log.info(
+                f"To update '{module}' run 'nf-core modules {repo_flag}{branch_flag}update {module}'. To force reinstallation use '--force'"
             )
+            return False
 
-            # Download module files
-            if not self.download_module_file(module, version, modules_repo, install_folder, module_dir):
-                exit_value = False
-                continue
+        if self.sha:
+            version = self.sha
+        elif self.prompt:
+            try:
+                version = nf_core.modules.module_utils.prompt_module_version_sha(
+                    module,
+                    installed_sha=current_entry["git_sha"] if not current_entry is None else None,
+                    modules_repo=self.modules_repo,
+                )
+            except SystemError as e:
+                log.error(e)
+                return False
+        else:
+            # Fetch the latest commit for the module
+            try:
+                git_log = get_module_git_log(module, modules_repo=self.modules_repo, per_page=1, page_nbr=1)
+            except UserWarning:
+                log.error(f"Was unable to fetch version of module '{module}'")
+                return False
+            version = git_log[0]["git_sha"]
+
+        if self.force:
+            log.info(f"Removing installed version of '{self.modules_repo.name}/{module}'")
+            self.clear_module_dir(module, module_dir)
+
+        log.info(f"{'Rei' if self.force else 'I'}nstalling '{self.modules_repo.name}/{module}'")
+        log.debug(f"Installing module '{module}' at modules hash {version} from {self.modules_repo.name}")
+
+        # Download module files
+        if not self.download_module_file(module, version, self.modules_repo, install_folder, module_dir):
+            return False
 
-            # Update module.json with newly installed module
-            self.update_modules_json(modules_json, modules_repo.name, module, version)
-        return exit_value
+        # Update module.json with newly installed module
+        self.update_modules_json(modules_json, self.modules_repo.name, module, version)
+        return True

nf_core/modules/modules_repo.py

@@ -29,7 +29,6 @@ def __init__(self, repo="nf-core/modules", branch="master"):
 
         self.owner, self.repo = self.name.split("/")
         self.modules_file_tree = {}
-        self.modules_current_hash = None
         self.modules_avail_module_names = []
 
     def verify_modules_repo(self):
@@ -65,7 +64,6 @@ def get_modules_file_tree(self):
         Fetch the file list from the repo, using the GitHub API
 
         Sets self.modules_file_tree
-             self.modules_current_hash
              self.modules_avail_module_names
         """
         api_url = "https://api.github.com/repos/{}/git/trees/{}?recursive=1".format(self.name, self.branch)
@@ -80,7 +78,6 @@ def get_modules_file_tree(self):
         result = r.json()
         assert result["truncated"] == False
 
-        self.modules_current_hash = result["sha"]
         self.modules_file_tree = result["tree"]
         for f in result["tree"]:
             if f["path"].startswith(f"modules/") and f["path"].endswith("/main.nf") and "/test/" not in f["path"]:

nf_core/modules/update.py

@@ -31,6 +31,9 @@ def update(self, module):
         # Verify that 'modules.json' is consistent with the installed modules
         self.modules_json_up_to_date()
 
+        tool_config = nf_core.utils.load_tools_config()
+        update_config = tool_config.get("update", {})
+
         if not self.update_all:
             # Get the available modules
             try:
@@ -55,33 +58,94 @@ def update(self, module):
                     style=nf_core.utils.nfcore_question_style,
                 ).unsafe_ask()
 
+            sha = self.sha
+            if module in update_config.get(self.modules_repo.name, {}):
+                config_entry = update_config[self.modules_repo.name].get(module)
+                if config_entry is not None and config_entry is not True:
+                    if config_entry is False:
+                        log.error("Module's update entry in '.nf-core.yml' is set to False")
+                        return False
+                    elif isinstance(config_entry, str):
+                        if self.sha:
+                            log.warning(
+                                "Found entry in '.nf-core.yml' for module "
+                                "which will override version specified with '--sha'"
+                            )
+                        sha = config_entry
+                    else:
+                        log.error("Module's update entry in '.nf-core.yml' is of wrong type")
+                        return False
+
             # Check that the supplied name is an available module
             if module and module not in self.modules_repo.modules_avail_module_names:
                 log.error("Module '{}' not found in list of available modules.".format(module))
                 log.info("Use the command 'nf-core modules list remote' to view available software")
                 return False
 
-            repos_and_modules = [(self.modules_repo, module)]
+            repos_mods_shas = [(self.modules_repo, module, sha)]
+
         else:
             if module:
                 raise UserWarning("You cannot specify a module and use the '--all' flag at the same time")
 
             self.get_pipeline_modules()
-            repos_and_modules = [
-                (ModulesRepo(repo=repo_name), modules) for repo_name, modules in self.module_names.items()
+
+            # Filter out modules that should not be updated or assign versions if there are any
+            skipped_repos = []
+            skipped_modules = []
+            repos_mods_shas = {}
+            for repo_name, modules in self.module_names.items():
+                if repo_name not in update_config or update_config[repo_name] is True:
+                    repos_mods_shas[repo_name] = []
+                    for module in modules:
+                        repos_mods_shas[repo_name].append((module, self.sha))
+                elif isinstance(update_config[repo_name], dict):
+                    repo_config = update_config[repo_name]
+                    repos_mods_shas[repo_name] = []
+                    for module in modules:
+                        if module not in repo_config or repo_config[module] is True:
+                            repos_mods_shas[repo_name].append((module, self.sha))
+                        elif isinstance(repo_config[module], str):
+                            # If a string is given it is the commit SHA to which we should update to
+                            custom_sha = repo_config[module]
+                            repos_mods_shas[repo_name].append((module, custom_sha))
+                        else:
+                            # Otherwise the entry must be 'False' and we should ignore the module
+                            skipped_modules.append(f"{repo_name}/{module}")
+                elif isinstance(update_config[repo_name], str):
+                    # If a string is given it is the commit SHA to which we should update to
+                    custom_sha = update_config[repo_name]
+                    repos_mods_shas[repo_name] = []
+                    for module in modules:
+                        repos_mods_shas[repo_name].append((module, custom_sha))
+                else:
+                    skipped_repos.append(repo_name)
+
+            if skipped_repos:
+                skipped_str = "', '".join(skipped_repos)
+                log.info(f"Skipping modules in repositor{'y' if len(skipped_repos) == 1 else 'ies'}: '{skipped_str}'")
+
+            if skipped_modules:
+                skipped_str = "', '".join(skipped_modules)
+                log.info(f"Skipping module{'' if len(skipped_modules) == 1 else 's'}: '{skipped_str}'")
+
+            repos_mods_shas = [
+                (ModulesRepo(repo=repo_name), mods_shas) for repo_name, mods_shas in repos_mods_shas.items()
             ]
-            # Load the modules file trees
-            for repo, _ in repos_and_modules:
+
+            for repo, _ in repos_mods_shas:
                 repo.get_modules_file_tree()
-            repos_and_modules = [(repo, module) for repo, modules in repos_and_modules for module in modules]
+
+            # Flatten the list
+            repos_mods_shas = [(repo, mod, sha) for repo, mods_shas in repos_mods_shas for mod, sha in mods_shas]
 
         # Load 'modules.json'
         modules_json = self.load_modules_json()
         if not modules_json:
             return False
 
         exit_value = True
-        for modules_repo, module in repos_and_modules:
+        for modules_repo, module, sha in repos_mods_shas:
             if not module_exist_in_repo(module, modules_repo):
                 warn_msg = f"Module '{module}' not found in remote '{modules_repo.name}' ({modules_repo.branch})"
                 if self.update_all:
@@ -101,8 +165,8 @@ def update(self, module):
             # Compute the module directory
             module_dir = os.path.join(self.dir, "modules", *install_folder, module)
 
-            if self.sha:
-                version = self.sha
+            if sha:
+                version = sha
             elif self.prompt:
                 try:
                     version = nf_core.modules.module_utils.prompt_module_version_sha(
@@ -135,9 +199,7 @@ def update(self, module):
                     continue
 
             log.info(f"Updating '{modules_repo.name}/{module}'")
-            log.debug(
-                f"Updating module '{module}' to {modules_repo.modules_current_hash} from {self.modules_repo.name}"
-            )
+            log.debug(f"Updating module '{module}' to {version} from {modules_repo.name}")
 
             log.debug(f"Removing old version of module '{module}'")
             self.clear_module_dir(module, module_dir)

nf_core/utils.py

@@ -771,7 +771,9 @@ def load_tools_config(dir="."):
     except FileNotFoundError:
         log.debug(f"No tools config file found: {config_fn}")
         return {}
-
+    if tools_config is None:
+        # If the file is empty
+        return {}
     return tools_config