Merge branch 'dev' into remove-avid-duplicates

nf_core/components/components_command.py

@@ -9,7 +9,7 @@
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
 
-from .components_utils import get_repo_type
+from .components_utils import get_repo_info
 
 log = logging.getLogger(__name__)
 
@@ -27,17 +27,31 @@ def __init__(self, component_type, dir, remote_url=None, branch=None, no_pull=Fa
         self.dir = dir
         self.modules_repo = ModulesRepo(remote_url, branch, no_pull, hide_progress)
         self.hide_progress = hide_progress
-        self.default_modules_path = Path("modules", "nf-core")
-        self.default_tests_path = Path("tests", "modules", "nf-core")
-        self.default_subworkflows_path = Path("subworkflows", "nf-core")
-        self.default_subworkflows_tests_path = Path("tests", "subworkflows", "nf-core")
+        self._configure_repo_and_paths()
+
+    def _configure_repo_and_paths(self, nf_dir_req=True):
+        """
+        Determine the repo type and set some default paths.
+        If this is a modules repo, determine the org_path too.
+
+        Args:
+            nf_dir_req (bool, optional): Whether this command requires being run in the nf-core modules repo or a nf-core pipeline repository. Defaults to True.
+        """
         try:
             if self.dir:
-                self.dir, self.repo_type = get_repo_type(self.dir)
+                self.dir, self.repo_type, self.org = get_repo_info(self.dir, use_prompt=nf_dir_req)
             else:
                 self.repo_type = None
-        except LookupError as e:
-            raise UserWarning(e)
+                self.org = ""
+        except UserWarning:
+            if nf_dir_req:
+                raise
+            self.repo_type = None
+            self.org = ""
+        self.default_modules_path = Path("modules", self.org)
+        self.default_tests_path = Path("tests", "modules", self.org)
+        self.default_subworkflows_path = Path("subworkflows", self.org)
+        self.default_subworkflows_tests_path = Path("tests", "subworkflows", self.org)
 
     def get_local_components(self):
         """

nf_core/components/components_create.py

@@ -88,7 +88,7 @@ def collect_name_prompt(name, component_type):
         return name
 
 
-def get_component_dirs(component_type, repo_type, directory, name, supername, subname, new_dir, force_overwrite):
+def get_component_dirs(component_type, repo_type, directory, org, name, supername, subname, new_dir, force_overwrite):
     """Given a directory and a tool/subtool or subworkflow, set the file paths and check if they already exist
 
     Returns dict: keys are relative paths to template files, vals are target paths.
@@ -117,8 +117,8 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su
         file_paths[os.path.join(component_type, "main.nf")] = component_file
 
     if repo_type == "modules":
-        software_dir = os.path.join(directory, component_type, "nf-core", new_dir)
-        test_dir = os.path.join(directory, "tests", component_type, "nf-core", new_dir)
+        software_dir = os.path.join(directory, component_type, org, new_dir)
+        test_dir = os.path.join(directory, "tests", component_type, org, new_dir)
 
         # Check if module/subworkflow directories exist already
         if os.path.exists(software_dir) and not force_overwrite:
@@ -128,15 +128,15 @@ def get_component_dirs(component_type, repo_type, directory, name, supername, su
 
         if component_type == "modules":
             # If a subtool, check if there is a module called the base tool name already
-            parent_tool_main_nf = os.path.join(directory, component_type, "nf-core", supername, "main.nf")
-            parent_tool_test_nf = os.path.join(directory, component_type, "nf-core", supername, "main.nf")
+            parent_tool_main_nf = os.path.join(directory, component_type, org, supername, "main.nf")
+            parent_tool_test_nf = os.path.join(directory, component_type, org, supername, "main.nf")
             if subname and os.path.exists(parent_tool_main_nf):
                 raise UserWarning(f"Module '{parent_tool_main_nf}' exists already, cannot make subtool '{name}'")
             if subname and os.path.exists(parent_tool_test_nf):
                 raise UserWarning(f"Module '{parent_tool_test_nf}' exists already, cannot make subtool '{name}'")
 
             # If no subtool, check that there isn't already a tool/subtool
-            tool_glob = glob.glob(f"{os.path.join(directory, component_type, 'nf-core', supername)}/*/main.nf")
+            tool_glob = glob.glob(f"{os.path.join(directory, component_type, org, supername)}/*/main.nf")
             if not subname and tool_glob:
                 raise UserWarning(f"Module subtool '{tool_glob[0]}' exists already, cannot make tool '{name}'")
 

nf_core/components/components_utils.py

@@ -11,38 +11,25 @@
 log = logging.getLogger(__name__)
 
 
-def get_repo_type(dir, repo_type=None, use_prompt=True):
+def get_repo_info(directory, use_prompt=True):
     """
     Determine whether this is a pipeline repository or a clone of
     nf-core/modules
     """
     # Verify that the pipeline dir exists
-    if dir is None or not os.path.exists(dir):
-        raise UserWarning(f"Could not find directory: {dir}")
+    if directory is None or not Path(directory).is_dir():
+        raise UserWarning(f"Could not find directory: {directory}")
 
     # Try to find the root directory
-    base_dir = os.path.abspath(dir)
-    config_path_yml = os.path.join(base_dir, ".nf-core.yml")
-    config_path_yaml = os.path.join(base_dir, ".nf-core.yaml")
-    while (
-        not os.path.exists(config_path_yml)
-        and not os.path.exists(config_path_yaml)
-        and base_dir != os.path.dirname(base_dir)
-    ):
-        base_dir = os.path.dirname(base_dir)
-        config_path_yml = os.path.join(base_dir, ".nf-core.yml")
-        config_path_yaml = os.path.join(base_dir, ".nf-core.yaml")
-        # Reset dir if we found the config file (will be an absolute path)
-        if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml):
-            dir = base_dir
+    base_dir = nf_core.utils.determine_base_dir(directory)
 
     # Figure out the repository type from the .nf-core.yml config file if we can
-    tools_config = nf_core.utils.load_tools_config(dir)
+    config_fn, tools_config = nf_core.utils.load_tools_config(base_dir)
     repo_type = tools_config.get("repository_type", None)
 
     # If not set, prompt the user
     if not repo_type and use_prompt:
-        log.warning("Can't find a '.nf-core.yml' file that defines 'repository_type'")
+        log.warning("'repository_type' not defined in %s", config_fn.name)
         repo_type = questionary.select(
             "Is this repository an nf-core pipeline or a fork of nf-core/modules?",
             choices=[
@@ -53,11 +40,11 @@ def get_repo_type(dir, repo_type=None, use_prompt=True):
         ).unsafe_ask()
 
         # Save the choice in the config file
-        log.info("To avoid this prompt in the future, add the 'repository_type' key to a root '.nf-core.yml' file.")
+        log.info(f"To avoid this prompt in the future, add the 'repository_type' key to your {config_fn.name} file.")
         if rich.prompt.Confirm.ask("[bold][blue]?[/] Would you like me to add this config now?", default=True):
-            with open(os.path.join(dir, ".nf-core.yml"), "a+") as fh:
+            with open(config_fn, "a+") as fh:
                 fh.write(f"repository_type: {repo_type}\n")
-                log.info("Config added to '.nf-core.yml'")
+                log.info(f"Config added to '{config_fn.name}'")
 
     # Not set and not allowed to ask
     elif not repo_type:
@@ -67,8 +54,30 @@ def get_repo_type(dir, repo_type=None, use_prompt=True):
     if not repo_type in ["pipeline", "modules"]:
         raise UserWarning(f"Invalid repository type: '{repo_type}'")
 
+    # Check for org if modules repo
+    org = None
+    if repo_type == "pipeline":
+        org = ""
+    elif repo_type == "modules":
+        org = tools_config.get("org_path", None)
+        if org is None:
+            log.warning("Organisation path not defined in %s [key: org_path]", config_fn.name)
+            org = questionary.text(
+                "What is the organisation path under which modules and subworkflows are stored?",
+                default="nf-core",
+                style=nf_core.utils.nfcore_question_style,
+            ).unsafe_ask()
+            log.info("To avoid this prompt in the future, add the 'org_path' key to a root '%s' file.", config_fn.name)
+            if rich.prompt.Confirm.ask("[bold][blue]?[/] Would you like me to add this config now?", default=True):
+                with open(os.path.join(dir, ".nf-core.yml"), "a+") as fh:
+                    fh.write(f"org_path: {org}\n")
+                    log.info(f"Config added to '{config_fn.name}'")
+
+        if not org:
+            raise UserWarning("Organisation path could not be established")
+
     # It was set on the command line, return what we were given
-    return [dir, repo_type]
+    return [base_dir, repo_type, org]
 
 
 def prompt_component_version_sha(component_name, component_type, modules_repo, installed_sha=None):

nf_core/components/info.py

@@ -15,7 +15,6 @@
 from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import NF_CORE_MODULES_REMOTE
-from nf_core.modules.modules_utils import get_repo_type
 
 log = logging.getLogger(__name__)
 
@@ -70,15 +69,6 @@ def __init__(
         self.remote_location = None
         self.local = None
 
-        # Quietly check if this is a pipeline or not
-        if pipeline_dir:
-            try:
-                pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False)
-                log.debug(f"Found {repo_type} repo: {pipeline_dir}")
-            except UserWarning as e:
-                log.debug(f"Only showing remote info: {e}")
-                pipeline_dir = None
-
         if self.repo_type == "pipeline":
             # Check modules directory structure
             if self.component_type == "modules":
@@ -90,6 +80,13 @@ def __init__(
             self.modules_json = None
         self.component = self.init_mod_name(component_name)
 
+    def _configure_repo_and_paths(self, nf_dir_req=False):
+        """
+        Override the default with nf_dir_req set to False to allow
+        info to be run from anywhere and still return remote info
+        """
+        return super()._configure_repo_and_paths(nf_dir_req)
+
     def init_mod_name(self, component):
         """
         Makes sure that we have a module/subworkflow name before proceeding.
@@ -109,7 +106,8 @@ def init_mod_name(self, component):
                         self.modules_repo.remote_url
                     )
                     components = [
-                        component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components
+                        component if directory == self.modules_repo.repo_path else f"{directory}/{component}"
+                        for directory, component in components
                     ]
                     if components is None:
                         raise UserWarning(
@@ -178,7 +176,7 @@ def get_local_yaml(self):
 
             log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally")
         else:
-            component_base_path = Path(self.dir, self.component_type, "nf-core")
+            component_base_path = Path(self.dir, self.component_type, self.org)
             if self.component in os.listdir(component_base_path):
                 comp_dir = Path(component_base_path, self.component)
                 meta_fn = Path(comp_dir, "meta.yml")

nf_core/components/list.py

@@ -3,7 +3,6 @@
 
 import rich
 
-import nf_core.modules.modules_utils
 from nf_core.components.components_command import ComponentCommand
 from nf_core.modules.modules_json import ModulesJson
 from nf_core.modules.modules_repo import ModulesRepo
@@ -64,8 +63,7 @@ def pattern_msg(keywords):
             # Check that we are in a pipeline directory
 
             try:
-                _, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir)
-                if repo_type != "pipeline":
+                if self.repo_type != "pipeline":
                     raise UserWarning(
                         f"The command 'nf-core {self.component_type} list local' must be run from a pipeline directory.",
                     )

nf_core/components/update.py

@@ -87,7 +87,7 @@ def update(self, component=None, silent=False, updated=None, check_diff_exist=Tr
         if updated is None:
             updated = []
 
-        tool_config = nf_core.utils.load_tools_config(self.dir)
+        _, tool_config = nf_core.utils.load_tools_config(self.dir)
         self.update_config = tool_config.get("update", {})
 
         self._parameter_checks()
@@ -348,7 +348,10 @@ def get_single_component_info(self, component):
         components = self.modules_json.get_all_components(self.component_type).get(repo_url)
         if components is None:
             raise LookupError(f"No {self.component_type} installed from '{repo_url}'")
-        choices = [component if dir == "nf-core" else f"{dir}/{component}" for dir, component in components]
+
+        choices = [
+            component if directory == "nf-core" else f"{directory}/{component}" for directory, component in components
+        ]
 
         if component is None:
             component = questionary.autocomplete(

nf_core/create.py

@@ -442,12 +442,12 @@ def fix_linting(self):
             lint_config["readme"] = ["nextflow_badge"]
 
         # Add the lint content to the preexisting nf-core config
-        nf_core_yml = nf_core.utils.load_tools_config(self.outdir)
+        config_fn, nf_core_yml = nf_core.utils.load_tools_config(self.outdir)
         nf_core_yml["lint"] = lint_config
-        with open(self.outdir / ".nf-core.yml", "w") as fh:
+        with open(self.outdir / config_fn, "w") as fh:
             yaml.dump(nf_core_yml, fh, default_flow_style=False, sort_keys=False)
 
-        run_prettier_on_file(os.path.join(self.outdir, ".nf-core.yml"))
+        run_prettier_on_file(os.path.join(self.outdir, config_fn))
 
     def make_pipeline_logo(self):
         """Fetch a logo for the new pipeline from the nf-core website"""

nf_core/lint/__init__.py

@@ -246,7 +246,7 @@ def _load_lint_config(self):
 
         Add parsed config to the `self.lint_config` class attribute.
         """
-        tools_config = nf_core.utils.load_tools_config(self.wf_path)
+        _, tools_config = nf_core.utils.load_tools_config(self.wf_path)
         self.lint_config = tools_config.get("lint", {})
 
         # Check if we have any keys that don't match lint test names

nf_core/modules/bump_versions.py

@@ -58,8 +58,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         self.check_modules_structure()
 
         # Verify that this is not a pipeline
-        self.dir, repo_type = nf_core.modules.modules_utils.get_repo_type(self.dir)
-        if not repo_type == "modules":
+        if not self.repo_type == "modules":
             raise nf_core.modules.modules_utils.ModuleException(
                 "This command only works on the nf-core/modules repository, not on pipelines!"
             )
@@ -68,7 +67,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         _, nfcore_modules = nf_core.modules.modules_utils.get_installed_modules(self.dir)
 
         # Load the .nf-core.yml config
-        self.tools_config = nf_core.utils.load_tools_config(self.dir)
+        _, self.tools_config = nf_core.utils.load_tools_config(self.dir)
 
         # Prompt for module or all
         if module is None and not all_modules:

nf_core/modules/create.py

@@ -14,7 +14,6 @@
 
 import nf_core
 import nf_core.components.components_create
-import nf_core.modules.modules_utils
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
 
@@ -32,7 +31,6 @@ def __init__(
         force=False,
         conda_name=None,
         conda_version=None,
-        repo_type=None,
     ):
         super().__init__("modules", directory)
         self.directory = directory
@@ -45,7 +43,6 @@ def __init__(
         self.tool_conda_name = conda_name
         self.tool_conda_version = conda_version
         self.tool_licence = None
-        self.repo_type = repo_type
         self.tool_licence = ""
         self.tool_description = ""
         self.tool_doc_url = ""
@@ -85,11 +82,6 @@ def create(self):
         # Check modules directory structure
         self.check_modules_structure()
 
-        # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules
-        try:
-            self.directory, self.repo_type = nf_core.modules.modules_utils.get_repo_type(self.directory, self.repo_type)
-        except LookupError as e:
-            raise UserWarning(e)
         log.info(f"Repository type: [blue]{self.repo_type}")
         if self.directory != ".":
             log.info(f"Base directory: '{self.directory}'")
@@ -119,6 +111,7 @@ def create(self):
             self.component_type,
             self.repo_type,
             self.directory,
+            self.org,
             self.tool_name,
             self.tool,
             self.subtool,
@@ -144,13 +137,13 @@ def create(self):
                     pytest_modules_yml = yaml.safe_load(fh)
                 if self.subtool:
                     pytest_modules_yml[self.tool_name] = [
-                        f"modules/nf-core/{self.tool}/{self.subtool}/**",
-                        f"tests/modules/nf-core/{self.tool}/{self.subtool}/**",
+                        f"modules/{self.org}/{self.tool}/{self.subtool}/**",
+                        f"tests/modules/{self.org}/{self.tool}/{self.subtool}/**",
                     ]
                 else:
                     pytest_modules_yml[self.tool_name] = [
-                        f"modules/nf-core/{self.tool}/**",
-                        f"tests/modules/nf-core/{self.tool}/**",
+                        f"modules/{self.org}/{self.tool}/**",
+                        f"tests/modules/{self.org}/{self.tool}/**",
                     ]
                 pytest_modules_yml = dict(sorted(pytest_modules_yml.items()))
                 with open(os.path.join(self.directory, "tests", "config", "pytest_modules.yml"), "w") as fh:

nf_core/modules/lint/__init__.py

@@ -91,7 +91,7 @@ def __init__(
             modules_json.check_up_to_date()
             all_pipeline_modules = modules_json.get_all_components(self.component_type)
             if all_pipeline_modules is not None and self.modules_repo.remote_url in all_pipeline_modules:
-                module_dir = Path(self.dir, "modules", "nf-core")
+                module_dir = Path(self.dir, "modules", self.modules_repo.repo_path)
                 self.all_remote_modules = [
                     NFCoreModule(m[1], self.modules_repo.remote_url, module_dir / m[1], self.repo_type, Path(self.dir))
                     for m in all_pipeline_modules[self.modules_repo.remote_url]