From ed47c3f76e9292c355d86925c1246e46f8443cac Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Mar 2022 23:27:23 +0100 Subject: [PATCH 1/4] New modules command: nf-core modules info --- nf_core/__main__.py | 34 +++++- nf_core/modules/__init__.py | 1 + nf_core/modules/info.py | 190 ++++++++++++++++++++++++++++++++ nf_core/modules/install.py | 3 +- nf_core/modules/module_utils.py | 8 +- 5 files changed, 231 insertions(+), 5 deletions(-) create mode 100644 nf_core/modules/info.py diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b771a4a9f0..7ce4a18a06 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -45,7 +45,7 @@ "nf-core modules": [ { "name": "For pipelines", - "commands": ["list", "install", "update", "remove"], + "commands": ["list", "info", "install", "update", "remove"], }, { "name": "Developing new modules", @@ -612,6 +612,38 @@ def lint(ctx, tool, dir, key, all, local, passed): sys.exit(1) +# nf-core modules info +@modules.command() +@click.pass_context +@click.argument("tool", type=str, required=False, metavar=" or ") +@click.option( + "-d", + "--dir", + type=click.Path(exists=True), + default=".", + help="Pipeline directory. [dim]\[default: Current working directory][/]", +) +def info(ctx, tool, dir): + """ + Show developer usage information about a given module. + + Parses information from a module's [i]meta.yml[/] and renders help + on the command line. A handy equivalent to searching the + [link=https://nf-co.re/modules]nf-core website[/]. + + If run from a pipeline and a local copy of the module is found, the command + will print this usage info. + If not, usage from the remote modules repo will be shown. + """ + try: + module_info = nf_core.modules.ModuleInfo(dir, tool) + module_info.modules_repo = ctx.obj["modules_repo_obj"] + print(module_info.get_module_info()) + except UserWarning as e: + log.error(e) + sys.exit(1) + + # nf-core modules bump-versions @modules.command() @click.pass_context diff --git a/nf_core/modules/__init__.py b/nf_core/modules/__init__.py index dbce4bd915..f833d52564 100644 --- a/nf_core/modules/__init__.py +++ b/nf_core/modules/__init__.py @@ -8,3 +8,4 @@ from .install import ModuleInstall from .update import ModuleUpdate from .remove import ModuleRemove +from .info import ModuleInfo diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py new file mode 100644 index 0000000000..b9ed2d4348 --- /dev/null +++ b/nf_core/modules/info.py @@ -0,0 +1,190 @@ +import base64 +import logging +import os +import requests +import yaml + +from rich import box +from rich.text import Text +from rich.console import Group +from rich.panel import Panel +from rich.table import Table + +from .modules_command import ModuleCommand +from .module_utils import get_repo_type, get_installed_modules, get_module_git_log, module_exist_in_repo +from .modules_repo import ModulesRepo + +log = logging.getLogger(__name__) + + +class ModuleInfo(ModuleCommand): + def __init__(self, pipeline_dir, tool): + + self.module = tool + self.meta = None + self.local_path = None + self.remote_location = None + + # Quietly check if this is a pipeline or not + if pipeline_dir: + try: + pipeline_dir, repo_type = get_repo_type(pipeline_dir, use_prompt=False) + log.debug(f"Found {repo_type} repo: {pipeline_dir}") + except UserWarning as e: + log.debug(f"Only showing remote info: {e}") + pipeline_dir = None + + super().__init__(pipeline_dir) + + def get_module_info(self): + """Given the name of a module, parse meta.yml and print usage help.""" + + # Running with a local install, try to find the local meta + if self.dir: + self.meta = self.get_local_yaml() + + # Either failed locally or in remote mode + if not self.meta: + self.meta = self.get_remote_yaml() + + # Could not find the meta + if self.meta == False: + raise UserWarning(f"Could not find module '{self.module}'") + + return self.generate_module_info_help() + + def get_local_yaml(self): + """Attempt to get the meta.yml file from a locally installed module. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + + # Get installed modules + self.get_pipeline_modules() + + # Try to find and load the meta.yml file + module_base_path = f"{self.dir}/modules/" + for dir, mods in self.module_names.items(): + for mod in mods: + if mod == self.module: + mod_dir = os.path.join(module_base_path, dir, mod) + meta_fn = os.path.join(mod_dir, "meta.yml") + if os.path.exists(meta_fn): + with open(meta_fn, "r") as fh: + self.local_path = mod_dir + return yaml.safe_load(fh) + + log.debug(f"Module '{self.module}' meta.yml not found locally") + return False + + def get_remote_yaml(self): + """Attempt to get the meta.yml file from a remote repo. + + Returns: + dict or bool: Parsed meta.yml found, False otherwise + """ + # Fetch the remote repo information + self.modules_repo.get_modules_file_tree() + + # Check if our requested module is there + if self.module not in self.modules_repo.modules_avail_module_names: + return False + + # Get the remote path + meta_url = None + for file_dict in self.modules_repo.modules_file_tree: + if file_dict.get("path") == f"modules/{self.module}/meta.yml": + meta_url = file_dict.get("url") + + if not meta_url: + return False + + # Download and parse + log.debug(f"Attempting to fetch {meta_url}") + response = requests.get(meta_url) + result = response.json() + file_contents = base64.b64decode(result["content"]) + self.remote_location = self.modules_repo.name + return yaml.safe_load(file_contents) + + def generate_module_info_help(self): + """Take the parsed meta.yml and generate rich help. + + Returns: + rich renderable + """ + + renderables = [] + + # Intro panel + intro_text = Text() + if self.local_path: + intro_text.append(Text.from_markup(f"Location: [blue]{self.local_path}\n")) + elif self.remote_location: + intro_text.append( + Text.from_markup( + f":globe_with_meridians: Repository: [link=https://github.com/{self.remote_location}]{self.remote_location}[/]\n" + ) + ) + + if self.meta.get("tools"): + tools_strings = [] + for tool in self.meta["tools"]: + for tool_name, tool_meta in tool.items(): + tools_strings.append(f"[link={tool_meta['homepage']}]{tool_name}") + intro_text.append(Text.from_markup(f":wrench: Tools: {', '.join(tools_strings)}\n", style="dim")) + + if self.meta.get("description"): + intro_text.append(Text.from_markup(f":book: Description: {self.meta['description']}", style="dim")) + + renderables.append( + Panel( + intro_text, + title=f"[bold]Module: [green]{self.module}\n", + title_align="left", + ) + ) + + # Inputs + if self.meta.get("input"): + inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + inputs_table.add_column(":inbox_tray: Inputs") + inputs_table.add_column("Description") + inputs_table.add_column("Pattern", justify="right", style="green") + for input in self.meta["input"]: + for key, info in input.items(): + inputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + info["description"].replace("\n", " "), + info.get("pattern", ""), + ) + + renderables.append(inputs_table) + + # Outputs + if self.meta.get("output"): + outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0) + outputs_table.add_column(":outbox_tray: Outputs") + outputs_table.add_column("Description") + outputs_table.add_column("Pattern", justify="right", style="green") + for output in self.meta["output"]: + for key, info in output.items(): + outputs_table.add_row( + f"[orange1 on black] {key} [/][dim i] ({info['type']})", + info["description"].replace("\n", " "), + info.get("pattern", ""), + ) + + renderables.append(outputs_table) + + # Installation command + if self.remote_location: + cmd_base = "nf-core modules" + if self.remote_location != "nf-core/modules": + cmd_base = f"nf-core modules --github-repository {self.remote_location}" + renderables.append( + Text.from_markup(f"\n :computer: Installation command: [magenta]{cmd_base} install {self.module}\n") + ) + + return Group(*renderables) diff --git a/nf_core/modules/install.py b/nf_core/modules/install.py index c5c33cc2a8..673e1d7f4e 100644 --- a/nf_core/modules/install.py +++ b/nf_core/modules/install.py @@ -6,8 +6,7 @@ import nf_core.modules.module_utils from .modules_command import ModuleCommand -from .module_utils import get_installed_modules, get_module_git_log, module_exist_in_repo -from .modules_repo import ModulesRepo +from .module_utils import get_module_git_log, module_exist_in_repo log = logging.getLogger(__name__) diff --git a/nf_core/modules/module_utils.py b/nf_core/modules/module_utils.py index 1349f443e1..0782a9a705 100644 --- a/nf_core/modules/module_utils.py +++ b/nf_core/modules/module_utils.py @@ -340,7 +340,7 @@ def get_installed_modules(dir, repo_type="modules"): return local_modules, nfcore_modules -def get_repo_type(dir, repo_type=None): +def get_repo_type(dir, repo_type=None, use_prompt=True): """ Determine whether this is a pipeline repository or a clone of nf-core/modules @@ -370,7 +370,7 @@ def get_repo_type(dir, repo_type=None): repo_type = tools_config.get("repository_type", None) # If not set, prompt the user - if not repo_type: + if not repo_type and use_prompt: log.warning(f"Can't find a '.nf-core.yml' file that defines 'repository_type'") repo_type = questionary.select( "Is this repository an nf-core pipeline or a fork of nf-core/modules?", @@ -388,6 +388,10 @@ def get_repo_type(dir, repo_type=None): fh.write(f"repository_type: {repo_type}\n") log.info("Config added to '.nf-core.yml'") + # Not set and not allowed to ask + elif not repo_type: + raise UserWarning("Repository type could not be established") + # Check if it's a valid answer if not repo_type in ["pipeline", "modules"]: raise UserWarning(f"Invalid repository type: '{repo_type}'") From 11408e3712cf365cffc1c0ebc5a2f30c453667f6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Mon, 14 Mar 2022 23:33:24 +0100 Subject: [PATCH 2/4] Readme and changelog --- CHANGELOG.md | 1 + README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cfc6657d8..0fa099addb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ ### Modules +* New command `nf-core modules info` that prints nice documentation about a module to the terminal :sparkles: ([#1427](https://github.com/nf-core/tools/issues/1427)) * Linting a pipeline now fails instead of warning if a local copy of a module does not match the remote ([#1313](https://github.com/nf-core/tools/issues/1313)) * Fixed linting bugs where warning was incorrectly generated for: * `Module does not emit software version` diff --git a/README.md b/README.md index 0aefa5171f..a418150e26 100644 --- a/README.md +++ b/README.md @@ -943,7 +943,7 @@ to get more information. The `nf-core modules list` command provides the subcommands `remote` and `local` for listing modules installed in a remote repository and in the local pipeline respectively. Both subcommands come with the `--key ` option for filtering the modules by keywords. -### List remote modules +#### List remote modules To list all modules available on [nf-core/modules](https://github.com/nf-core/modules), you can use `nf-core modules list remote`, which will print all available modules to the terminal. @@ -975,7 +975,7 @@ INFO Modules available from nf-core/modules (master) └────────────────────────────────┘ ``` -### List installed modules +#### List installed modules To list modules installed in a local pipeline directory you can use `nf-core modules list local`. This will list the modules install in the current working directory by default. If you want to specify another directory, use the `--dir ` flag. @@ -1000,6 +1000,54 @@ INFO Modules installed in '.': └─────────────┴─────────────────┴─────────────┴────────────────────────────────────────────────────────┴────────────┘ ``` +## Show information about a module + +For quick help about how a module works, use `nf-core modules info `. +This shows documentation about the module on the command line, similar to what's available on the +[nf-core website](https://nf-co.re/modules). + +```console +$ nf-core modules info fastqc + + ,--./,-. + ___ __ __ __ ___ /,-._.--~\ + |\ | |__ __ / ` / \ |__) |__ } { + | \| | \__, \__/ | \ |___ \`-._,-`-, + `._,._,' + + nf-core/tools version 2.3.dev0 - https://nf-co.re + + +╭─ Module: fastqc ───────────────────────────────────────────────────────────────────────────────────────╮ +│ 🌐 Repository: nf-core/modules │ +│ 🔧 Tools: fastqc │ +│ 📖 Description: Run FastQC on sequenced reads │ +╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ╷ ╷ + 📥 Inputs │Description │Pattern +╺━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━╸ + meta (map) │Groovy Map containing sample information e.g. [ id:'test', single_end:false ] │ +╶──────────────┼──────────────────────────────────────────────────────────────────────────────────┼───────╴ + reads (file)│List of input FastQ files of size 1 and 2 for single-end and paired-end data, │ + │respectively. │ + ╵ ╵ + ╷ ╷ + 📤 Outputs │Description │ Pattern +╺━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━╸ + meta (map) │Groovy Map containing sample information e.g. [ id:'test', │ + │single_end:false ] │ +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + html (file) │FastQC report │*_{fastqc.html} +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + zip (file) │FastQC report archive │ *_{fastqc.zip} +╶─────────────────┼───────────────────────────────────────────────────────────────────────┼───────────────╴ + versions (file)│File containing software versions │ versions.yml + ╵ ╵ + + 💻 Installation command: nf-core modules install fastqc + +``` + ### Install modules in a pipeline You can install modules from [nf-core/modules](https://github.com/nf-core/modules) in your pipeline using `nf-core modules install`. From 728df851ff81014ec9186752de5acd7dab2df172 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 15 Mar 2022 20:06:24 +0100 Subject: [PATCH 3/4] Descriptions as markdown, fix search path for modules --- nf_core/modules/info.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index b9ed2d4348..5da2b0e239 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -7,6 +7,7 @@ from rich import box from rich.text import Text from rich.console import Group +from rich.markdown import Markdown from rich.panel import Panel from rich.table import Table @@ -65,12 +66,15 @@ def get_local_yaml(self): # Try to find and load the meta.yml file module_base_path = f"{self.dir}/modules/" + if self.repo_type == "modules": + module_base_path = f"{self.dir}/" for dir, mods in self.module_names.items(): for mod in mods: if mod == self.module: mod_dir = os.path.join(module_base_path, dir, mod) meta_fn = os.path.join(mod_dir, "meta.yml") if os.path.exists(meta_fn): + log.debug(f"Found local file: {meta_fn}") with open(meta_fn, "r") as fh: self.local_path = mod_dir return yaml.safe_load(fh) @@ -156,7 +160,7 @@ def generate_module_info_help(self): for key, info in input.items(): inputs_table.add_row( f"[orange1 on black] {key} [/][dim i] ({info['type']})", - info["description"].replace("\n", " "), + Markdown(info["description"]), info.get("pattern", ""), ) @@ -172,7 +176,7 @@ def generate_module_info_help(self): for key, info in output.items(): outputs_table.add_row( f"[orange1 on black] {key} [/][dim i] ({info['type']})", - info["description"].replace("\n", " "), + Markdown(info["description"]), info.get("pattern", ""), ) From f3d4937f5f8cd7506cf1930e113de3b274ea3aca Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Tue, 15 Mar 2022 20:11:37 +0100 Subject: [PATCH 4/4] Handle markdown conversion when description is None --- nf_core/modules/info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/modules/info.py b/nf_core/modules/info.py index 5da2b0e239..88a178546f 100644 --- a/nf_core/modules/info.py +++ b/nf_core/modules/info.py @@ -160,7 +160,7 @@ def generate_module_info_help(self): for key, info in input.items(): inputs_table.add_row( f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"]), + Markdown(info["description"] if info["description"] else ""), info.get("pattern", ""), ) @@ -176,7 +176,7 @@ def generate_module_info_help(self): for key, info in output.items(): outputs_table.add_row( f"[orange1 on black] {key} [/][dim i] ({info['type']})", - Markdown(info["description"]), + Markdown(info["description"] if info["description"] else ""), info.get("pattern", ""), )