Merge pull request #1391 from ewels/modules-create-type

Rewrite modules create repo type check.

CHANGELOG.md

@@ -7,6 +7,7 @@
 * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md`
 * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode.
 * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2)
+* Added `.nf-core.yml` file with `repository_type: pipeline` for modules commands
 * Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file
 * Remove pinned Node version in the GitHub Actions workflows, to fix errors with `markdownlint`
 * Add yamllint GitHub Action.
@@ -16,6 +17,7 @@
 ## General
 
 * Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379))
+* Made `nf-core modules create` detect repository type with explicit `.nf-core.yml` or `--repo-type`, instead of random readme stuff ([#1391](https://github.com/nf-core/tools/pull/1391))
 * Added a Gitpod environment and Dockerfile ([#1384](https://github.com/nf-core/tools/pull/1384))
     * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image.
     * Adds GH action to build and push Gitpod Docker image.

README.md

@@ -1116,11 +1116,17 @@ This command creates a new nf-core module from the nf-core module template.
 This ensures that your module follows the nf-core guidelines.
 The template contains extensive `TODO` messages to walk you through the changes you need to make to the template.
 
-You can create a new module using `nf-core modules create`. This will create the new module in the current working directory. To specify another directory, use `--dir <directory>`.
+You can create a new module using `nf-core modules create`.
 
-If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `<directory>` argument should be the path to the clone of your fork of the modules repository.
+This command can be used both when writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository,
+and also when creating local modules for a pipeline.
 
-Alternatively, if writing a more niche module that does not make sense to share, `<directory>` should be the path to your pipeline.
+Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory,
+set to either `pipeline` or `modules`.
+The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory.
+It will start in the current working directory, or whatever is specified with `--dir <directory>`.
+
+If you do not have a `.nf-core.yml` file you can specify the repository type with the `--repo-type` flag.
 
 The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files.
 

nf_core/__main__.py

@@ -520,7 +520,8 @@ def remove(ctx, dir, tool):
 @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information")
 @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist")
 @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use")
-def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name):
+@click.option("-r", "--repo-type", type=click.Choice(["pipeline", "modules"]), default=None, help="Type of repository")
+def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, repo_type):
     """
     Create a new DSL2 module from the nf-core template.
 
@@ -541,7 +542,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam
 
     # Run function
     try:
-        module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name)
+        module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name, repo_type)
         module_create.create()
     except UserWarning as e:
         log.critical(e)

nf_core/modules/bump_versions.py

@@ -15,8 +15,6 @@
 from rich.markdown import Markdown
 import rich
 from nf_core.utils import rich_force_colors
-import sys
-import yaml
 
 import nf_core.utils
 import nf_core.modules.module_utils
@@ -55,7 +53,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         self.show_up_to_date = show_uptodate
 
         # Verify that this is not a pipeline
-        repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
+        self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         if not repo_type == "modules":
             raise nf_core.modules.module_utils.ModuleException(
                 "This command only works on the nf-core/modules repository, not on pipelines!"
@@ -64,7 +62,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False):
         # Get list of all modules
         _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir)
 
-        # Load the .nf-core-tools.config
+        # Load the .nf-core.yml config
         self.tools_config = nf_core.utils.load_tools_config(self.dir)
 
         # Prompt for module or all

nf_core/modules/create.py

@@ -19,13 +19,22 @@
 import yaml
 
 import nf_core.utils
+import nf_core.modules.module_utils
 
 log = logging.getLogger(__name__)
 
 
 class ModuleCreate(object):
     def __init__(
-        self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None
+        self,
+        directory=".",
+        tool="",
+        author=None,
+        process_label=None,
+        has_meta=None,
+        force=False,
+        conda_name=None,
+        repo_type=None,
     ):
         self.directory = directory
         self.tool = tool
@@ -36,7 +45,7 @@ def __init__(
         self.subtool = None
         self.tool_conda_name = conda_name
         self.tool_licence = None
-        self.repo_type = None
+        self.repo_type = repo_type
         self.tool_licence = ""
         self.tool_description = ""
         self.tool_doc_url = ""
@@ -75,9 +84,12 @@ def create(self):
 
         # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules
         try:
-            self.repo_type = self.get_repo_type(self.directory)
+            self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type)
         except LookupError as e:
             raise UserWarning(e)
+        log.info(f"Repository type: [blue]{self.repo_type}")
+        if self.directory != ".":
+            log.info(f"Base directory: '{self.directory}'")
 
         log.info(
             "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. "
@@ -272,29 +284,6 @@ def render_template(self):
             template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn))
             os.chmod(dest_fn, template_stat.st_mode)
 
-    def get_repo_type(self, directory):
-        """
-        Determine whether this is a pipeline repository or a clone of
-        nf-core/modules
-        """
-        # Verify that the pipeline dir exists
-        if dir is None or not os.path.exists(directory):
-            raise UserWarning(f"Could not find directory: {directory}")
-
-        readme = os.path.join(directory, "README.md")
-        # Determine repository type
-        if os.path.exists(readme):
-            with open(readme) as fh:
-                if fh.readline().rstrip().startswith("# ![nf-core/modules]"):
-                    return "modules"
-                else:
-                    return "pipeline"
-        else:
-            raise UserWarning(
-                f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'"
-                " Please point to a valid directory."
-            )
-
     def get_module_dirs(self):
         """Given a directory and a tool/subtool, set the file paths and check if they already exist
 

nf_core/modules/lint/__init__.py

@@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand):
     def __init__(self, dir):
         self.dir = dir
         try:
-            self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
+            self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
         except LookupError as e:
             raise UserWarning(e)
 

nf_core/modules/module_utils.py

@@ -340,24 +340,49 @@ def get_installed_modules(dir, repo_type="modules"):
     return local_modules, nfcore_modules
 
 
-def get_repo_type(dir):
+def get_repo_type(dir, repo_type=None):
     """
     Determine whether this is a pipeline repository or a clone of
     nf-core/modules
     """
     # Verify that the pipeline dir exists
     if dir is None or not os.path.exists(dir):
-        raise LookupError("Could not find directory: {}".format(dir))
+        raise UserWarning(f"Could not find directory: {dir}")
+
+    # Try to find the root directory
+    base_dir = os.path.abspath(dir)
+    config_path_yml = os.path.join(base_dir, ".nf-core.yml")
+    config_path_yaml = os.path.join(base_dir, ".nf-core.yaml")
+    while (
+        not os.path.exists(config_path_yml)
+        and not os.path.exists(config_path_yaml)
+        and base_dir != os.path.dirname(base_dir)
+    ):
+        base_dir = os.path.dirname(base_dir)
+        config_path_yml = os.path.join(base_dir, ".nf-core.yml")
+        config_path_yaml = os.path.join(base_dir, ".nf-core.yaml")
+        # Reset dir if we found the config file (will be an absolute path)
+        if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml):
+            dir = base_dir
+
+    # Figure out the repository type from the .nf-core.yml config file if we can
+    tools_config = nf_core.utils.load_tools_config(dir)
+    if tools_config.get("repository_type") in ["pipeline", "modules"]:
+        if repo_type is not None and repo_type != tools_config["repository_type"]:
+            raise UserWarning(
+                f"'--repo-type {repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'"
+            )
+        return [dir, tools_config["repository_type"]]
 
-    # Determine repository type
-    if os.path.exists(os.path.join(dir, "README.md")):
-        with open(os.path.join(dir, "README.md")) as fh:
-            if fh.readline().rstrip().startswith("# ![nf-core/modules]"):
-                return "modules"
-            else:
-                return "pipeline"
-    else:
-        raise LookupError("Could not determine repository type of '{}'".format(dir))
+    # Could be set on the command line - throw an error if not
+    if not repo_type:
+        raise UserWarning(
+            f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{dir}'"
+            "\nPlease use the '--repo-type' flag or create '.nf-core.yml'"
+        )
+
+    # It was set on the command line, return what we were given
+    return [dir, repo_type]
 
 
 def verify_pipeline_dir(dir):

nf_core/modules/modules_command.py

@@ -29,7 +29,7 @@ def __init__(self, dir):
         self.module_names = []
         try:
             if self.dir:
-                self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
+                self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir)
             else:
                 self.repo_type = None
         except LookupError as e:

nf_core/pipeline-template/.nf-core.yml

@@ -0,0 +1 @@
+repository_type: pipeline

tests/test_modules.py

@@ -19,8 +19,8 @@ def create_modules_repo_dummy(tmp_dir):
     os.makedirs(os.path.join(root_dir, "tests", "config"))
     with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh:
         fh.writelines(["test:", "\n  - modules/test/**", "\n  - tests/modules/test/**"])
-    with open(os.path.join(root_dir, "README.md"), "w") as fh:
-        fh.writelines(["# ![nf-core/modules](docs/images/nfcore-modules_logo.png)", "\n"])
+    with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh:
+        fh.writelines(["repository_type: modules", "\n"])
 
     # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules
     module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False)
@@ -47,15 +47,10 @@ def setUp(self):
         self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True)
         self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True)
 
-        # TODO Remove comments once external repository to have same structure as nf-core/modules
-        # self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master")
-
         # Set up remove objects
         print("Setting up remove objects")
         self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir)
         self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir)
-        # TODO Remove comments once external repository to have same structure as nf-core/modules
-        # self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master")
 
         # Set up the nf-core/modules repo dummy
         self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir)