diff --git a/CHANGELOG.md b/CHANGELOG.md index c04d8cf3fb..96e993cdbc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * Removed mention of `--singularity_pull_docker_container` in pipeline `README.md` * Replaced equals with ~ in nf-core headers, to stop false positive unresolved conflict errors when committing with VSCode. * Add retry strategy for AWS megatests after releasing [nf-core/tower-action v2.2](https://github.com/nf-core/tower-action/releases/tag/v2.2) +* Added `.nf-core.yml` file with `repository_type: pipeline` for modules commands * Update igenomes path to the `BWAIndex` to fetch the whole `version0.6.0` folder instead of only the `genome.fa` file * Remove pinned Node version in the GitHub Actions workflows, to fix errors with `markdownlint` * Add yamllint GitHub Action. @@ -16,6 +17,7 @@ ## General * Updated `nf-core download` to work with latest DSL2 syntax for containers ([#1379](https://github.com/nf-core/tools/issues/1379)) +* Made `nf-core modules create` detect repository type with explicit `.nf-core.yml` or `--repo-type`, instead of random readme stuff ([#1391](https://github.com/nf-core/tools/pull/1391)) * Added a Gitpod environment and Dockerfile ([#1384](https://github.com/nf-core/tools/pull/1384)) * Adds conda, Nextflow, nf-core, pytest-workflow, mamba, and pip to base Gitpod Docker image. * Adds GH action to build and push Gitpod Docker image. diff --git a/README.md b/README.md index f9ae7f2253..404f990eba 100644 --- a/README.md +++ b/README.md @@ -1116,11 +1116,17 @@ This command creates a new nf-core module from the nf-core module template. This ensures that your module follows the nf-core guidelines. The template contains extensive `TODO` messages to walk you through the changes you need to make to the template. -You can create a new module using `nf-core modules create`. This will create the new module in the current working directory. To specify another directory, use `--dir `. +You can create a new module using `nf-core modules create`. -If writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, the `` argument should be the path to the clone of your fork of the modules repository. +This command can be used both when writing a module for the shared [nf-core/modules](https://github.com/nf-core/modules) repository, +and also when creating local modules for a pipeline. -Alternatively, if writing a more niche module that does not make sense to share, `` should be the path to your pipeline. +Which type of repository you are working in is detected by the `repository_type` flag in a `.nf-core.yml` file in the root directory, +set to either `pipeline` or `modules`. +The command will automatically look through parent directories for this file to set the root path, so that you can run the command in a subdirectory. +It will start in the current working directory, or whatever is specified with `--dir `. + +If you do not have a `.nf-core.yml` file you can specify the repository type with the `--repo-type` flag. The `nf-core modules create` command will prompt you with the relevant questions in order to create all of the necessary module files. diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 830f8bc14c..a92cb83adb 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -520,7 +520,8 @@ def remove(ctx, dir, tool): @click.option("-n", "--no-meta", is_flag=True, default=False, help="Don't use meta map for sample information") @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite any files if they already exist") @click.option("-c", "--conda-name", type=str, default=None, help="Name of the conda package to use") -def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name): +@click.option("-r", "--repo-type", type=click.Choice(["pipeline", "modules"]), default=None, help="Type of repository") +def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_name, repo_type): """ Create a new DSL2 module from the nf-core template. @@ -541,7 +542,7 @@ def create_module(ctx, tool, dir, author, label, meta, no_meta, force, conda_nam # Run function try: - module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name) + module_create = nf_core.modules.ModuleCreate(dir, tool, author, label, has_meta, force, conda_name, repo_type) module_create.create() except UserWarning as e: log.critical(e) diff --git a/nf_core/modules/bump_versions.py b/nf_core/modules/bump_versions.py index 4d24faa52b..d8b0450a69 100644 --- a/nf_core/modules/bump_versions.py +++ b/nf_core/modules/bump_versions.py @@ -15,8 +15,6 @@ from rich.markdown import Markdown import rich from nf_core.utils import rich_force_colors -import sys -import yaml import nf_core.utils import nf_core.modules.module_utils @@ -55,7 +53,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): self.show_up_to_date = show_uptodate # Verify that this is not a pipeline - repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) if not repo_type == "modules": raise nf_core.modules.module_utils.ModuleException( "This command only works on the nf-core/modules repository, not on pipelines!" @@ -64,7 +62,7 @@ def bump_versions(self, module=None, all_modules=False, show_uptodate=False): # Get list of all modules _, nfcore_modules = nf_core.modules.module_utils.get_installed_modules(self.dir) - # Load the .nf-core-tools.config + # Load the .nf-core.yml config self.tools_config = nf_core.utils.load_tools_config(self.dir) # Prompt for module or all diff --git a/nf_core/modules/create.py b/nf_core/modules/create.py index a9c071c4ce..c4371853ed 100644 --- a/nf_core/modules/create.py +++ b/nf_core/modules/create.py @@ -19,13 +19,22 @@ import yaml import nf_core.utils +import nf_core.modules.module_utils log = logging.getLogger(__name__) class ModuleCreate(object): def __init__( - self, directory=".", tool="", author=None, process_label=None, has_meta=None, force=False, conda_name=None + self, + directory=".", + tool="", + author=None, + process_label=None, + has_meta=None, + force=False, + conda_name=None, + repo_type=None, ): self.directory = directory self.tool = tool @@ -36,7 +45,7 @@ def __init__( self.subtool = None self.tool_conda_name = conda_name self.tool_licence = None - self.repo_type = None + self.repo_type = repo_type self.tool_licence = "" self.tool_description = "" self.tool_doc_url = "" @@ -75,9 +84,12 @@ def create(self): # Check whether the given directory is a nf-core pipeline or a clone of nf-core/modules try: - self.repo_type = self.get_repo_type(self.directory) + self.directory, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.directory, self.repo_type) except LookupError as e: raise UserWarning(e) + log.info(f"Repository type: [blue]{self.repo_type}") + if self.directory != ".": + log.info(f"Base directory: '{self.directory}'") log.info( "[yellow]Press enter to use default values [cyan bold](shown in brackets)[/] [yellow]or type your own responses. " @@ -272,29 +284,6 @@ def render_template(self): template_stat = os.stat(os.path.join(os.path.dirname(nf_core.__file__), "module-template", template_fn)) os.chmod(dest_fn, template_stat.st_mode) - def get_repo_type(self, directory): - """ - Determine whether this is a pipeline repository or a clone of - nf-core/modules - """ - # Verify that the pipeline dir exists - if dir is None or not os.path.exists(directory): - raise UserWarning(f"Could not find directory: {directory}") - - readme = os.path.join(directory, "README.md") - # Determine repository type - if os.path.exists(readme): - with open(readme) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise UserWarning( - f"This directory does not look like a clone of nf-core/modules or an nf-core pipeline: '{directory}'" - " Please point to a valid directory." - ) - def get_module_dirs(self): """Given a directory and a tool/subtool, set the file paths and check if they already exist diff --git a/nf_core/modules/lint/__init__.py b/nf_core/modules/lint/__init__.py index 892cacf8de..e81eea7f85 100644 --- a/nf_core/modules/lint/__init__.py +++ b/nf_core/modules/lint/__init__.py @@ -71,7 +71,7 @@ class ModuleLint(ModuleCommand): def __init__(self, dir): self.dir = dir try: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) except LookupError as e: raise UserWarning(e) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 4e3a0c5ffe..1ec163198e 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -340,24 +340,49 @@ def get_installed_modules(dir, repo_type="modules"): return local_modules, nfcore_modules -def get_repo_type(dir): +def get_repo_type(dir, repo_type=None): """ Determine whether this is a pipeline repository or a clone of nf-core/modules """ # Verify that the pipeline dir exists if dir is None or not os.path.exists(dir): - raise LookupError("Could not find directory: {}".format(dir)) + raise UserWarning(f"Could not find directory: {dir}") + + # Try to find the root directory + base_dir = os.path.abspath(dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + while ( + not os.path.exists(config_path_yml) + and not os.path.exists(config_path_yaml) + and base_dir != os.path.dirname(base_dir) + ): + base_dir = os.path.dirname(base_dir) + config_path_yml = os.path.join(base_dir, ".nf-core.yml") + config_path_yaml = os.path.join(base_dir, ".nf-core.yaml") + # Reset dir if we found the config file (will be an absolute path) + if os.path.exists(config_path_yml) or os.path.exists(config_path_yaml): + dir = base_dir + + # Figure out the repository type from the .nf-core.yml config file if we can + tools_config = nf_core.utils.load_tools_config(dir) + if tools_config.get("repository_type") in ["pipeline", "modules"]: + if repo_type is not None and repo_type != tools_config["repository_type"]: + raise UserWarning( + f"'--repo-type {repo_type}' conflicts with [i][red]repository_type[/]: [blue]pipeline[/][/] in '{os.path.relpath(config_path_yml)}'" + ) + return [dir, tools_config["repository_type"]] - # Determine repository type - if os.path.exists(os.path.join(dir, "README.md")): - with open(os.path.join(dir, "README.md")) as fh: - if fh.readline().rstrip().startswith("# ![nf-core/modules]"): - return "modules" - else: - return "pipeline" - else: - raise LookupError("Could not determine repository type of '{}'".format(dir)) + # Could be set on the command line - throw an error if not + if not repo_type: + raise UserWarning( + f"Can't find a '.nf-core.yml' file with 'repository_type' set to 'pipeline' or 'modules': '{dir}'" + "\nPlease use the '--repo-type' flag or create '.nf-core.yml'" + ) + + # It was set on the command line, return what we were given + return [dir, repo_type] def verify_pipeline_dir(dir): diff --git a/nf_core/modules/modules_command.py b/nf_core/modules/modules_command.py index 866bffabb9..58198740b6 100644 --- a/nf_core/modules/modules_command.py +++ b/nf_core/modules/modules_command.py @@ -29,7 +29,7 @@ def __init__(self, dir): self.module_names = [] try: if self.dir: - self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) + self.dir, self.repo_type = nf_core.modules.module_utils.get_repo_type(self.dir) else: self.repo_type = None except LookupError as e: diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml new file mode 100644 index 0000000000..3805dc81c1 --- /dev/null +++ b/nf_core/pipeline-template/.nf-core.yml @@ -0,0 +1 @@ +repository_type: pipeline diff --git a/tests/test_modules.py b/tests/test_modules.py index fb59537c62..cfa3408e69 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -19,8 +19,8 @@ def create_modules_repo_dummy(tmp_dir): os.makedirs(os.path.join(root_dir, "tests", "config")) with open(os.path.join(root_dir, "tests", "config", "pytest_modules.yml"), "w") as fh: fh.writelines(["test:", "\n - modules/test/**", "\n - tests/modules/test/**"]) - with open(os.path.join(root_dir, "README.md"), "w") as fh: - fh.writelines(["# ![nf-core/modules](docs/images/nfcore-modules_logo.png)", "\n"]) + with open(os.path.join(root_dir, ".nf-core.yml"), "w") as fh: + fh.writelines(["repository_type: modules", "\n"]) # bpipe is a valid package on bioconda that is very unlikely to ever be added to nf-core/modules module_create = nf_core.modules.ModuleCreate(root_dir, "bpipe/test", "@author", "process_medium", False, False) @@ -47,15 +47,10 @@ def setUp(self): self.mods_install = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=False, force=True) self.mods_install_alt = nf_core.modules.ModuleInstall(self.pipeline_dir, prompt=True, force=True) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_install_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") - # Set up remove objects print("Setting up remove objects") self.mods_remove = nf_core.modules.ModuleRemove(self.pipeline_dir) self.mods_remove_alt = nf_core.modules.ModuleRemove(self.pipeline_dir) - # TODO Remove comments once external repository to have same structure as nf-core/modules - # self.mods_remove_alt.modules_repo = nf_core.modules.ModulesRepo(repo="ewels/nf-core-modules", branch="master") # Set up the nf-core/modules repo dummy self.nfcore_modules = create_modules_repo_dummy(self.tmp_dir)