feat: Install subworkflows with modules from different remotes

nf_core/__main__.py

@@ -54,7 +54,7 @@
     subworkflows_test,
     subworkflows_update,
 )
-from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_REMOTE
 from nf_core.pipelines.download import DownloadError
 from nf_core.utils import check_if_outdated, nfcore_logo, rich_force_colors, setup_nfcore_dir
 

nf_core/components/components_utils.py

@@ -1,24 +1,18 @@
 import logging
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 import questionary
 import requests
 import rich.prompt
-
-if TYPE_CHECKING:
-    from nf_core.modules.modules_repo import ModulesRepo
+import yaml
 
 import nf_core.utils
+from nf_core.modules.modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
-# Constants for the nf-core/modules repo used throughout the module files
-NF_CORE_MODULES_NAME = "nf-core"
-NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git"
-NF_CORE_MODULES_DEFAULT_BRANCH = "master"
-
 
 def get_repo_info(directory: Path, use_prompt: Optional[bool] = True) -> Tuple[Path, Optional[str], str]:
     """
@@ -143,12 +137,15 @@ def prompt_component_version_sha(
     return git_sha
 
 
-def get_components_to_install(subworkflow_dir: Union[str, Path]) -> Tuple[List[str], List[str]]:
+def get_components_to_install(
+    subworkflow_dir: Union[str, Path],
+) -> Tuple[List[Dict[str, Optional[str]]], List[Dict[str, Optional[str]]]]:
     """
     Parse the subworkflow main.nf file to retrieve all imported modules and subworkflows.
     """
-    modules = []
-    subworkflows = []
+    modules: Dict[str, Dict[str, Optional[str]]] = {}
+    subworkflows: Dict[str, Dict[str, Optional[str]]] = {}
+
     with open(Path(subworkflow_dir, "main.nf")) as fh:
         for line in fh:
             regex = re.compile(
@@ -159,10 +156,38 @@ def get_components_to_install(subworkflow_dir: Union[str, Path]) -> Tuple[List[s
                 name, link = match.groups()
                 if link.startswith("../../../"):
                     name_split = name.lower().split("_")
-                    modules.append("/".join(name_split))
+                    component_name = "/".join(name_split)
+                    component_dict: Dict[str, Optional[str]] = {
+                        "name": component_name,
+                    }
+                    modules[component_name] = component_dict
                 elif link.startswith("../"):
-                    subworkflows.append(name.lower())
-    return modules, subworkflows
+                    component_name = name.lower()
+                    component_dict = {"name": component_name}
+                    subworkflows[component_name] = component_dict
+
+    if (sw_meta := Path(subworkflow_dir, "meta.yml")).exists():
+        with open(sw_meta) as fh:
+            meta = yaml.safe_load(fh)
+            if "components" in meta:
+                components = meta["components"]
+                for component in components:
+                    if isinstance(component, dict):
+                        component_name = list(component.keys())[0].lower()
+                        branch = component[component_name].get("branch")
+                        git_remote = component[component_name]["git_remote"]
+                        modules_repo = ModulesRepo(git_remote, branch=branch)
+                        current_comp_dict = subworkflows if component_name in subworkflows else modules
+
+                        component_dict = {
+                            "org_path": modules_repo.repo_path,
+                            "git_remote": git_remote,
+                            "branch": branch,
+                        }
+
+                        current_comp_dict[component_name].update(component_dict)
+
+    return list(modules.values()), list(subworkflows.values())
 
 
 def get_biotools_id(tool_name) -> str:

nf_core/components/constants.py

@@ -0,0 +1,4 @@
+# Constants for the nf-core/modules repo used throughout the module files
+NF_CORE_MODULES_NAME = "nf-core"
+NF_CORE_MODULES_REMOTE = "https://github.com/nf-core/modules.git"
+NF_CORE_MODULES_DEFAULT_BRANCH = "master"

nf_core/components/info.py

@@ -15,7 +15,7 @@
 
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
-from nf_core.components.components_utils import NF_CORE_MODULES_REMOTE
+from nf_core.components.constants import NF_CORE_MODULES_REMOTE
 from nf_core.modules.modules_json import ModulesJson
 
 log = logging.getLogger(__name__)

nf_core/components/install.py

@@ -1,7 +1,7 @@
 import logging
 import os
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import questionary
 from rich import print
@@ -15,11 +15,14 @@
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.components.components_utils import (
-    NF_CORE_MODULES_NAME,
     get_components_to_install,
     prompt_component_version_sha,
 )
+from nf_core.components.constants import (
+    NF_CORE_MODULES_NAME,
+)
 from nf_core.modules.modules_json import ModulesJson
+from nf_core.modules.modules_repo import ModulesRepo
 
 log = logging.getLogger(__name__)
 
@@ -38,15 +41,33 @@ def __init__(
         installed_by: Optional[List[str]] = None,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
+        self.current_remote = ModulesRepo(remote_url, branch)
+        self.branch = branch
         self.force = force
         self.prompt = prompt
         self.sha = sha
+        self.current_sha = sha
         if installed_by is not None:
             self.installed_by = installed_by
         else:
             self.installed_by = [self.component_type]
 
-    def install(self, component: str, silent: bool = False) -> bool:
+    def install(self, component: Union[str, Dict[str, str]], silent: bool = False) -> bool:
+        if isinstance(component, dict):
+            # Override modules_repo when the component to install is a dependency from a subworkflow.
+            remote_url = component.get("git_remote", self.current_remote.remote_url)
+            branch = component.get("branch", self.branch)
+            self.modules_repo = ModulesRepo(remote_url, branch)
+            component = component["name"]
+
+        if self.current_remote is None:
+            self.current_remote = self.modules_repo
+
+        if self.current_remote.remote_url == self.modules_repo.remote_url and self.sha is not None:
+            self.current_sha = self.sha
+        else:
+            self.current_sha = None
+
         if self.repo_type == "modules":
             log.error(f"You cannot install a {component} in a clone of nf-core/modules")
             return False
@@ -70,8 +91,8 @@ def install(self, component: str, silent: bool = False) -> bool:
             return False
 
         # Verify SHA
-        if not self.modules_repo.verify_sha(self.prompt, self.sha):
-            err_msg = f"SHA '{self.sha}' is not a valid commit SHA for the repository '{self.modules_repo.remote_url}'"
+        if not self.modules_repo.verify_sha(self.prompt, self.current_sha):
+            err_msg = f"SHA '{self.current_sha}' is not a valid commit SHA for the repository '{self.modules_repo.remote_url}'"
             log.error(err_msg)
             return False
 
@@ -114,7 +135,7 @@ def install(self, component: str, silent: bool = False) -> bool:
             modules_json.update(self.component_type, self.modules_repo, component, current_version, self.installed_by)
             return False
         try:
-            version = self.get_version(component, self.sha, self.prompt, current_version, self.modules_repo)
+            version = self.get_version(component, self.current_sha, self.prompt, current_version, self.modules_repo)
         except UserWarning as e:
             log.error(e)
             return False
@@ -199,15 +220,17 @@ def collect_and_verify_name(
         if component is None:
             component = questionary.autocomplete(
                 f"{'Tool' if self.component_type == 'modules' else 'Subworkflow'} name:",
-                choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.sha)),
+                choices=sorted(modules_repo.get_avail_components(self.component_type, commit=self.current_sha)),
                 style=nf_core.utils.nfcore_question_style,
             ).unsafe_ask()
 
         if component is None:
             return ""
 
         # Check that the supplied name is an available module/subworkflow
-        if component and component not in modules_repo.get_avail_components(self.component_type, commit=self.sha):
+        if component and component not in modules_repo.get_avail_components(
+            self.component_type, commit=self.current_sha
+        ):
             log.error(f"{self.component_type[:-1].title()} '{component}' not found in available {self.component_type}")
             print(
                 Panel(
@@ -223,9 +246,10 @@ def collect_and_verify_name(
 
             raise ValueError
 
-        if not modules_repo.component_exists(component, self.component_type, commit=self.sha):
-            warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})"
-            log.warning(warn_msg)
+        if self.current_remote.remote_url == modules_repo.remote_url:
+            if not modules_repo.component_exists(component, self.component_type, commit=self.current_sha):
+                warn_msg = f"{self.component_type[:-1].title()} '{component}' not found in remote '{modules_repo.remote_url}' ({modules_repo.branch})"
+                log.warning(warn_msg)
 
         return component
 

nf_core/components/update.py

@@ -41,6 +41,8 @@ def __init__(
         limit_output=False,
     ):
         super().__init__(component_type, pipeline_dir, remote_url, branch, no_pull)
+        self.current_remote = ModulesRepo(remote_url, branch)
+        self.branch = branch
         self.force = force
         self.prompt = prompt
         self.sha = sha
@@ -92,6 +94,13 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
         Returns:
             (bool): True if the update was successful, False otherwise.
         """
+        if isinstance(component, dict):
+            # Override modules_repo when the component to install is a dependency from a subworkflow.
+            remote_url = component.get("git_remote", self.current_remote.remote_url)
+            branch = component.get("branch", self.branch)
+            self.modules_repo = ModulesRepo(remote_url, branch)
+            component = component["name"]
+
         self.component = component
         if updated is None:
             updated = []
@@ -882,7 +891,17 @@ def get_components_to_update(self, component):
         if self.component_type == "modules":
             # All subworkflow names in the installed_by section of a module are subworkflows using this module
             # We need to update them too
-            subworkflows_to_update = [subworkflow for subworkflow in installed_by if subworkflow != self.component_type]
+            git_remote = self.current_remote.remote_url
+            for subworkflow in installed_by:
+                if subworkflow != component:
+                    for remote_url, content in mods_json["repos"].items():
+                        if all_subworkflows := content.get("subworkflows"):
+                            for _, details in all_subworkflows.items():
+                                if subworkflow in details:
+                                    git_remote = remote_url
+                if subworkflow != self.component_type:
+                    subworkflows_to_update.append({"name": subworkflow, "git_remote": git_remote})
+
         elif self.component_type == "subworkflows":
             for repo, repo_content in mods_json["repos"].items():
                 for component_type, dir_content in repo_content.items():
@@ -893,9 +912,9 @@ def get_components_to_update(self, component):
                             # We need to update it too
                             if component in comp_content["installed_by"]:
                                 if component_type == "modules":
-                                    modules_to_update.append(comp)
+                                    modules_to_update.append({"name": comp, "git_remote": repo, "org_path": dir})
                                 elif component_type == "subworkflows":
-                                    subworkflows_to_update.append(comp)
+                                    subworkflows_to_update.append({"name": comp, "git_remote": repo, "org_path": dir})
 
         return modules_to_update, subworkflows_to_update
 
@@ -910,7 +929,7 @@ def update_linked_components(
         Update modules and subworkflows linked to the component being updated.
         """
         for s_update in subworkflows_to_update:
-            if s_update in updated:
+            if s_update["name"] in updated:
                 continue
             original_component_type, original_update_all = self._change_component_type("subworkflows")
             self.update(
@@ -922,7 +941,7 @@ def update_linked_components(
             self._reset_component_type(original_component_type, original_update_all)
 
         for m_update in modules_to_update:
-            if m_update in updated:
+            if m_update["name"] in updated:
                 continue
             original_component_type, original_update_all = self._change_component_type("modules")
             try:
@@ -945,28 +964,42 @@ def update_linked_components(
     def manage_changes_in_linked_components(self, component, modules_to_update, subworkflows_to_update):
         """Check for linked components added or removed in the new subworkflow version"""
         if self.component_type == "subworkflows":
-            subworkflow_directory = Path(self.directory, self.component_type, self.modules_repo.repo_path, component)
+            org_path = self.current_remote.repo_path
+
+            subworkflow_directory = Path(self.directory, self.component_type, org_path, component)
             included_modules, included_subworkflows = get_components_to_install(subworkflow_directory)
             # If a module/subworkflow has been removed from the subworkflow
             for module in modules_to_update:
-                if module not in included_modules:
-                    log.info(f"Removing module '{module}' which is not included in '{component}' anymore.")
+                module_name = module["name"]
+                included_modules_names = [m["name"] for m in included_modules]
+                if module_name not in included_modules_names:
+                    log.info(f"Removing module '{module_name}' which is not included in '{component}' anymore.")
                     remove_module_object = ComponentRemove("modules", self.directory)
-                    remove_module_object.remove(module, removed_by=component)
+                    remove_module_object.remove(module_name, removed_by=component)
             for subworkflow in subworkflows_to_update:
-                if subworkflow not in included_subworkflows:
-                    log.info(f"Removing subworkflow '{subworkflow}' which is not included in '{component}' anymore.")
+                subworkflow_name = subworkflow["name"]
+                included_subworkflow_names = [m["name"] for m in included_subworkflows]
+                if subworkflow_name not in included_subworkflow_names:
+                    log.info(
+                        f"Removing subworkflow '{subworkflow_name}' which is not included in '{component}' anymore."
+                    )
                     remove_subworkflow_object = ComponentRemove("subworkflows", self.directory)
-                    remove_subworkflow_object.remove(subworkflow, removed_by=component)
+                    remove_subworkflow_object.remove(subworkflow_name, removed_by=component)
             # If a new module/subworkflow is included in the subworklfow and wasn't included before
             for module in included_modules:
-                if module not in modules_to_update:
-                    log.info(f"Installing newly included module '{module}' for '{component}'")
+                module_name = module["name"]
+                module["git_remote"] = module.get("git_remote", self.current_remote.remote_url)
+                module["branch"] = module.get("branch", self.branch)
+                if module_name not in modules_to_update:
+                    log.info(f"Installing newly included module '{module_name}' for '{component}'")
                     install_module_object = ComponentInstall(self.directory, "modules", installed_by=component)
                     install_module_object.install(module, silent=True)
             for subworkflow in included_subworkflows:
-                if subworkflow not in subworkflows_to_update:
-                    log.info(f"Installing newly included subworkflow '{subworkflow}' for '{component}'")
+                subworkflow_name = subworkflow["name"]
+                subworkflow["git_remote"] = subworkflow.get("git_remote", self.current_remote.remote_url)
+                subworkflow["branch"] = subworkflow.get("branch", self.branch)
+                if subworkflow_name not in subworkflows_to_update:
+                    log.info(f"Installing newly included subworkflow '{subworkflow_name}' for '{component}'")
                     install_subworkflow_object = ComponentInstall(
                         self.directory, "subworkflows", installed_by=component
                     )
@@ -985,3 +1018,5 @@ def _reset_component_type(self, original_component_type, original_update_all):
         self.component_type = original_component_type
         self.modules_json.pipeline_components = None
         self.update_all = original_update_all
+        if self.current_remote is None:
+            self.current_remote = self.modules_repo