From cb06dc9379bfbe9e628407ee4f51d32a78f963ec Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 15 Apr 2021 12:12:05 +0200 Subject: [PATCH 01/45] Added questionary select list for releases --- nf_core/__main__.py | 4 ++-- nf_core/download.py | 25 +++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9ce7cfcfd..5f0c2007b 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -201,8 +201,8 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download @nf_core_cli.command(help_priority=3) -@click.argument("pipeline", metavar="") -@click.option("-r", "--release", type=str, help="Pipeline release") +@click.argument("pipeline", required=False, metavar="") +@click.option("-r", "--release", is_flag=True, help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-c", diff --git a/nf_core/download.py b/nf_core/download.py index a5e0a88e6..d9638e027 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -76,7 +76,7 @@ class DownloadWorkflow(object): def __init__( self, pipeline=None, - release=None, + release_flag=False, outdir=None, compress_type="tar.gz", force=False, @@ -85,7 +85,8 @@ def __init__( parallel_downloads=4, ): self.pipeline = pipeline - self.release = release + self.release_flag = release_flag + self.release = None self.outdir = outdir self.output_filename = None self.compress_type = compress_type @@ -121,6 +122,11 @@ def download_workflow(self): style=nf_core.utils.nfcore_question_style, ).ask() + # Prompts user for release tag if '-r' was set + if self.release_flag: + release_tags = self.fetch_release_tags() + self.release = questionary.select("Select release:", release_tags).ask() + # Get workflow details try: self.fetch_workflow_details(wfs) @@ -179,6 +185,21 @@ def download_workflow(self): log.info("Compressing download..") self.compress_download() + def fetch_release_tags(self): + # Fetch releases from github api + releases_url = "https://api.github.com/repos/nf-core/{}/releases".format(self.pipeline) + response = requests.get(releases_url) + + # Filter out the release tags and sort them + release_tags = map(lambda release: release.get("tag_name", None), response.json()) + release_tags = filter(lambda tag: tag != None, release_tags) + release_tags = list(release_tags) + if len(release_tags) == 0: + log.error("Unable to find any releases!") + sys.exit(1) + release_tags = sorted(release_tags, key=lambda tag: tuple(tag.split(".")), reverse=True) + return release_tags + def fetch_workflow_details(self, wfs): """Fetches details of a nf-core workflow to download. From 519dd18d1f95c813c8ba636cc96292845d552b2d Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 15 Apr 2021 14:14:46 +0200 Subject: [PATCH 02/45] Added confirmation prompt for image download and some docs --- nf_core/__main__.py | 25 +++++++++++++++++++++---- nf_core/download.py | 29 ++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 5f0c2007b..d17c0eec0 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -3,6 +3,7 @@ from click.types import File from rich import print +from rich.prompt import Confirm import click import logging import os @@ -200,6 +201,15 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download +def confirm_container_download(ctx, opts, value): + """Confirm choice of container""" + if value != "none": + is_satisfied = Confirm.ask(f"Should {value} image be downloaded?") + if not is_satisfied: + value = 'none' + return value + + @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") @click.option("-r", "--release", is_flag=True, help="Pipeline release") @@ -212,16 +222,23 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all help="Archive compression type", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") -@click.option("-s", "--singularity", is_flag=True, default=False, help="Download singularity images") @click.option( - "-c", + "-C", + "--container", + type=click.Choice(['none', 'singularity']), + default="none", + callback=confirm_container_download, + help="Download images", +) +@click.option( + "-s", "--singularity-cache", is_flag=True, default=False, help="Don't copy images to the output directory, don't set 'singularity.cacheDir' in workflow", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") -def download(pipeline, release, outdir, compress, force, singularity, singularity_cache, parallel_downloads): +def download(pipeline, release, outdir, compress, force, container, singularity_cache, parallel_downloads): """ Download a pipeline, nf-core/configs and pipeline singularity images. @@ -229,7 +246,7 @@ def download(pipeline, release, outdir, compress, force, singularity, singularit workflow to use relative paths to the configs and singularity images. """ dl = nf_core.download.DownloadWorkflow( - pipeline, release, outdir, compress, force, singularity, singularity_cache, parallel_downloads + pipeline, release, outdir, compress, force, container, singularity_cache, parallel_downloads ) dl.download_workflow() diff --git a/nf_core/download.py b/nf_core/download.py index d9638e027..b0210ded3 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -80,7 +80,7 @@ def __init__( outdir=None, compress_type="tar.gz", force=False, - singularity=False, + container='none', singularity_cache_only=False, parallel_downloads=4, ): @@ -93,13 +93,13 @@ def __init__( if self.compress_type == "none": self.compress_type = None self.force = force - self.singularity = singularity + self.singularity = container == "singularity" self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads # Sanity checks if self.singularity_cache_only and not self.singularity: - log.error("Command has '--singularity-cache' set, but not '--singularity'") + log.error("Command has '--singularity-cache' set, but '--container' is 'none'") sys.exit(1) self.wf_name = None @@ -110,11 +110,11 @@ def __init__( def download_workflow(self): """Starts a nf-core workflow download.""" - # Fetches remote workflows + # Fetch remote workflows wfs = nf_core.list.Workflows() wfs.get_remote_workflows() - # Prompts user if pipeline name was not specified + # Prompt user if pipeline name was not specified if self.pipeline is None: self.pipeline = questionary.autocomplete( "Pipeline name:", @@ -122,10 +122,13 @@ def download_workflow(self): style=nf_core.utils.nfcore_question_style, ).ask() - # Prompts user for release tag if '-r' was set + # Prompt user for release tag if '--release' was set if self.release_flag: - release_tags = self.fetch_release_tags() - self.release = questionary.select("Select release:", release_tags).ask() + try: + release_tags = self.fetch_release_tags() + except LookupError: + sys.exit(1) + self.release = questionary.select("Select release:", choices=release_tags).ask() # Get workflow details try: @@ -186,6 +189,14 @@ def download_workflow(self): self.compress_download() def fetch_release_tags(self): + """Fetches tag names of pipeline releases from github + + Returns: + release_tags (list[str]): Returns list of release tags + + Raises: + LookupError, if no releases were found + """ # Fetch releases from github api releases_url = "https://api.github.com/repos/nf-core/{}/releases".format(self.pipeline) response = requests.get(releases_url) @@ -196,7 +207,7 @@ def fetch_release_tags(self): release_tags = list(release_tags) if len(release_tags) == 0: log.error("Unable to find any releases!") - sys.exit(1) + raise LookupError release_tags = sorted(release_tags, key=lambda tag: tuple(tag.split(".")), reverse=True) return release_tags From 77808a45ab2311de9af646b4f83bc23f8c9b316e Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 15 Apr 2021 14:34:02 +0200 Subject: [PATCH 03/45] Added prompt for singularity caching (and 'negated' prompt for image download) --- nf_core/__main__.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d17c0eec0..0478abcbe 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -203,12 +203,19 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download def confirm_container_download(ctx, opts, value): """Confirm choice of container""" - if value != "none": - is_satisfied = Confirm.ask(f"Should {value} image be downloaded?") - if not is_satisfied: - value = 'none' + if value == None: + should_download = Confirm.ask(f"Should singularity image be downloaded?") + if should_download: + value = "singularity" + else: + value = "none" return value +def confirm_singularity_cache(ctx, opts, value): + """Confirm that singularity image should be cached""" + if not value: + return Confirm.ask(f"Should singularity image be cached?") + return value @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") @@ -226,7 +233,7 @@ def confirm_container_download(ctx, opts, value): "-C", "--container", type=click.Choice(['none', 'singularity']), - default="none", + default=None, callback=confirm_container_download, help="Download images", ) @@ -234,6 +241,7 @@ def confirm_container_download(ctx, opts, value): "-s", "--singularity-cache", is_flag=True, + callback=confirm_singularity_cache, default=False, help="Don't copy images to the output directory, don't set 'singularity.cacheDir' in workflow", ) From f53b942289dc5a7f1105e7c944b10433b557a8bb Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 15 Apr 2021 14:44:53 +0200 Subject: [PATCH 04/45] Added prompt for compression type --- nf_core/__main__.py | 3 +-- nf_core/download.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 0478abcbe..fc01d25f0 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -224,8 +224,7 @@ def confirm_singularity_cache(ctx, opts, value): @click.option( "-c", "--compress", - type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), - default="tar.gz", + is_flag=True, help="Archive compression type", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") diff --git a/nf_core/download.py b/nf_core/download.py index b0210ded3..470f6b8a9 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -78,7 +78,7 @@ def __init__( pipeline=None, release_flag=False, outdir=None, - compress_type="tar.gz", + compress=False, force=False, container='none', singularity_cache_only=False, @@ -89,9 +89,8 @@ def __init__( self.release = None self.outdir = outdir self.output_filename = None - self.compress_type = compress_type - if self.compress_type == "none": - self.compress_type = None + self.compress = compress + self.compress_type = None self.force = force self.singularity = container == "singularity" self.singularity_cache_only = singularity_cache_only @@ -183,6 +182,15 @@ def download_workflow(self): self.find_container_images() self.get_singularity_images() + # If '--compress' flag was set, ask user what compression type to be used + if self.compress: + self.compress_type = questionary.select( + "Choose compression type:", + choices=["none", "tar.gz", "tar.bz2", "zip",] + ).ask() + if self.compress_type == "none": + self.compress_type = None + # Compress into an archive if self.compress_type is not None: log.info("Compressing download..") From 958e95bc055f779175f36139d3971dcdcc084873 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Thu, 15 Apr 2021 16:03:02 +0200 Subject: [PATCH 05/45] Added checking for 'export NXF_SINGULARITY_CACHEDIR' in bashrc --- nf_core/__main__.py | 4 +++- nf_core/download.py | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index fc01d25f0..2452717e7 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -211,12 +211,14 @@ def confirm_container_download(ctx, opts, value): value = "none" return value + def confirm_singularity_cache(ctx, opts, value): """Confirm that singularity image should be cached""" if not value: return Confirm.ask(f"Should singularity image be cached?") return value + @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") @click.option("-r", "--release", is_flag=True, help="Pipeline release") @@ -231,7 +233,7 @@ def confirm_singularity_cache(ctx, opts, value): @click.option( "-C", "--container", - type=click.Choice(['none', 'singularity']), + type=click.Choice(["none", "singularity"]), default=None, callback=confirm_container_download, help="Download images", diff --git a/nf_core/download.py b/nf_core/download.py index 470f6b8a9..f9a81e90c 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -18,6 +18,7 @@ import tarfile import concurrent.futures from rich.progress import BarColumn, DownloadColumn, TransferSpeedColumn, Progress +from rich.prompt import Confirm from zipfile import ZipFile import nf_core @@ -80,7 +81,7 @@ def __init__( outdir=None, compress=False, force=False, - container='none', + container="none", singularity_cache_only=False, parallel_downloads=4, ): @@ -139,8 +140,18 @@ def download_workflow(self): "Pipeline release: '{}'".format(self.release), "Pull singularity containers: '{}'".format("Yes" if self.singularity else "No"), ] - if self.singularity and os.environ.get("NXF_SINGULARITY_CACHEDIR"): - summary_log.append("Using '$NXF_SINGULARITY_CACHEDIR': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"])) + if self.singularity: + export_in_file = ( + os.popen('grep -c "export NXF_SINGULARITY_CACHEDIR" ~/.bashrc').read().strip("\n") != "0" + ) + if not export_in_file: + append_to_file = Confirm.ask("Add 'export NXF_SINGULARITY_CACHEDIR' to .bashrc?") + if append_to_file: + os.system('echo "export NXF_SINGULARITY_CACHEDIR" >> ~/.bashrc') + if os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: + summary_log.append( + "Using '$NXF_SINGULARITY_CACHEDIR': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) + ) # Set an output filename now that we have the outdir if self.compress_type is not None: @@ -186,7 +197,12 @@ def download_workflow(self): if self.compress: self.compress_type = questionary.select( "Choose compression type:", - choices=["none", "tar.gz", "tar.bz2", "zip",] + choices=[ + "none", + "tar.gz", + "tar.bz2", + "zip", + ], ).ask() if self.compress_type == "none": self.compress_type = None From 897bae3691df8a262988b95e106681dee7874f77 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 16 Apr 2021 08:56:33 +0200 Subject: [PATCH 06/45] Added version selection list for launch --- nf_core/download.py | 4 +--- nf_core/launch.py | 44 ++++++++++++++++++++++++++++++++++++++++++-- nf_core/list.py | 2 +- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index f9a81e90c..0395ada18 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -141,9 +141,7 @@ def download_workflow(self): "Pull singularity containers: '{}'".format("Yes" if self.singularity else "No"), ] if self.singularity: - export_in_file = ( - os.popen('grep -c "export NXF_SINGULARITY_CACHEDIR" ~/.bashrc').read().strip("\n") != "0" - ) + export_in_file = os.popen('grep -c "export NXF_SINGULARITY_CACHEDIR" ~/.bashrc').read().strip("\n") != "0" if not export_in_file: append_to_file = Confirm.ask("Add 'export NXF_SINGULARITY_CACHEDIR' to .bashrc?") if append_to_file: diff --git a/nf_core/launch.py b/nf_core/launch.py index ce571f373..5e304892d 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -14,8 +14,9 @@ import re import subprocess import webbrowser +import requests -import nf_core.schema, nf_core.utils +import nf_core.schema, nf_core.utils, nf_core.download log = logging.getLogger(__name__) @@ -176,6 +177,18 @@ def get_pipeline_schema(self): # Assume nf-core if no org given if self.pipeline.count("/") == 0: self.nextflow_cmd = "nextflow run nf-core/{}".format(self.pipeline) + + if not self.pipeline_revision: + check_for_releases = Confirm.ask("Would you like to select a specific release?") + if check_for_releases: + try: + release_tags = self.try_fetch_release_tags() + self.pipeline_revision = questionary.select( + "Please select a release:", choices=release_tags + ).ask() + except LookupError: + pass + # Add revision flag to commands if set if self.pipeline_revision: self.nextflow_cmd += " -r {}".format(self.pipeline_revision) @@ -184,10 +197,11 @@ def get_pipeline_schema(self): try: self.schema_obj.get_schema_path(self.pipeline, revision=self.pipeline_revision) self.schema_obj.load_lint_schema() - except AssertionError: + except AssertionError as a: # No schema found # Check that this was actually a pipeline if self.schema_obj.pipeline_dir is None or not os.path.exists(self.schema_obj.pipeline_dir): + log.info(f"dir: {a}") log.error("Could not find pipeline: {} ({})".format(self.pipeline, self.schema_obj.pipeline_dir)) return False if not os.path.exists(os.path.join(self.schema_obj.pipeline_dir, "nextflow.config")) and not os.path.exists( @@ -208,6 +222,32 @@ def get_pipeline_schema(self): log.error("Could not build pipeline schema: {}".format(e)) return False + def try_fetch_release_tags(self): + """Tries to fetch tag names of pipeline releases from github + + Returns: + release_tags (list[str]): Returns list of release tags + + Raises: + LookupError, if no releases were found + """ + # Fetch releases from github api + releases_url = "https://api.github.com/repos/nf-core/{}/releases".format(self.pipeline) + response = requests.get(releases_url) + if not response.ok: + log.error(f"Unable to find any release tags for {self.pipeline}. Will try to continue launch.") + raise LookupError + + # Filter out the release tags and sort them + release_tags = map(lambda release: release.get("tag_name", None), response.json()) + release_tags = filter(lambda tag: tag != None, release_tags) + release_tags = list(release_tags) + if len(release_tags) == 0: + log.error(f"Unable to find any release tags for {self.pipeline}. Will try to continue launch.") + raise LookupError + release_tags = sorted(release_tags, key=lambda tag: tuple(tag.split(".")), reverse=True) + return release_tags + def set_schema_inputs(self): """ Take the loaded schema and set the defaults as the input parameters diff --git a/nf_core/list.py b/nf_core/list.py index 327ae1a87..3278d8e1f 100644 --- a/nf_core/list.py +++ b/nf_core/list.py @@ -71,7 +71,7 @@ def get_local_wf(workflow, revision=None): log.info("Downloading workflow: {} ({})".format(workflow, revision)) pull_cmd = f"nextflow pull {workflow}" if revision is not None: - pull_cmd += f"-r {revision}" + pull_cmd += f" -r {revision}" nf_pull_output = nf_core.utils.nextflow_cmd(pull_cmd) local_wf = LocalWorkflow(workflow) local_wf.get_local_nf_workflow_details() From 89ff0add111d0aacbd9475e683a1ed86e3cc32ee Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 16 Apr 2021 13:43:43 +0200 Subject: [PATCH 07/45] Added path prompt --- nf_core/download.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 0395ada18..683678188 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -18,7 +18,7 @@ import tarfile import concurrent.futures from rich.progress import BarColumn, DownloadColumn, TransferSpeedColumn, Progress -from rich.prompt import Confirm +from rich.prompt import Confirm, Prompt from zipfile import ZipFile import nf_core @@ -145,7 +145,14 @@ def download_workflow(self): if not export_in_file: append_to_file = Confirm.ask("Add 'export NXF_SINGULARITY_CACHEDIR' to .bashrc?") if append_to_file: - os.system('echo "export NXF_SINGULARITY_CACHEDIR" >> ~/.bashrc') + path = Prompt.ask("Specify the path: ") + try: + with open(os.path.expanduser("~/.bashrc"), "a") as f: + f.write(f'export NXF_SINGULARITY_CACHEDIR={path}\n') + log.info("Successfully wrote to ~/.bashrc") + except FileNotFoundError: + log.error("Unable to find ~/.bashrc") + sys.exit(1) if os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: summary_log.append( "Using '$NXF_SINGULARITY_CACHEDIR': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) From bf4b64a7c411e7d874d96f2e3afb392c6bcfb0c2 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 16 Apr 2021 14:15:38 +0200 Subject: [PATCH 08/45] Changed to only prompt when options are not specified --- nf_core/__main__.py | 26 +++------------------ nf_core/download.py | 57 +++++++++++++++++++++++++++++---------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 2452717e7..b0948fea7 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -201,32 +201,16 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all # nf-core download -def confirm_container_download(ctx, opts, value): - """Confirm choice of container""" - if value == None: - should_download = Confirm.ask(f"Should singularity image be downloaded?") - if should_download: - value = "singularity" - else: - value = "none" - return value - - -def confirm_singularity_cache(ctx, opts, value): - """Confirm that singularity image should be cached""" - if not value: - return Confirm.ask(f"Should singularity image be cached?") - return value @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") -@click.option("-r", "--release", is_flag=True, help="Pipeline release") +@click.option("-r", "--release", help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-c", "--compress", - is_flag=True, + type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type", ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @@ -234,16 +218,12 @@ def confirm_singularity_cache(ctx, opts, value): "-C", "--container", type=click.Choice(["none", "singularity"]), - default=None, - callback=confirm_container_download, help="Download images", ) @click.option( "-s", "--singularity-cache", - is_flag=True, - callback=confirm_singularity_cache, - default=False, + type=click.Choice(["yes", "no"]), help="Don't copy images to the output directory, don't set 'singularity.cacheDir' in workflow", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") diff --git a/nf_core/download.py b/nf_core/download.py index 683678188..bddf0598b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -77,24 +77,32 @@ class DownloadWorkflow(object): def __init__( self, pipeline=None, - release_flag=False, + release=None, outdir=None, - compress=False, + compress_type=None, force=False, container="none", singularity_cache_only=False, parallel_downloads=4, ): self.pipeline = pipeline - self.release_flag = release_flag - self.release = None + self.release = release self.outdir = outdir self.output_filename = None - self.compress = compress - self.compress_type = None + self.compress_type = compress_type + if self.compress_type is None: + self.compress_type = self._confirm_compression() + if self.compress_type == "none": + self.compress_type = None + self.force = force + + if container is None: + container = self._confirm_container_download() self.singularity = container == "singularity" self.singularity_cache_only = singularity_cache_only + if self.singularity_cache_only is None and self.singularity: + self.singularity_cache_only = self._confirm_singularity_cache() self.parallel_downloads = parallel_downloads # Sanity checks @@ -108,6 +116,27 @@ def __init__( self.nf_config = dict() self.containers = list() + def _confirm_compression(self): + return questionary.select( + "Choose compression type:", + choices=[ + "none", + "tar.gz", + "tar.bz2", + "zip", + ], + ).ask() + + def _confirm_container_download(self): + should_download = Confirm.ask(f"Should singularity image be downloaded?") + if should_download: + return "singularity" + else: + return "none" + + def _confirm_singularity_cache(self): + return Confirm.ask(f"Should singularity image be cached?") + def download_workflow(self): """Starts a nf-core workflow download.""" # Fetch remote workflows @@ -123,7 +152,7 @@ def download_workflow(self): ).ask() # Prompt user for release tag if '--release' was set - if self.release_flag: + if self.release is None: try: release_tags = self.fetch_release_tags() except LookupError: @@ -148,7 +177,7 @@ def download_workflow(self): path = Prompt.ask("Specify the path: ") try: with open(os.path.expanduser("~/.bashrc"), "a") as f: - f.write(f'export NXF_SINGULARITY_CACHEDIR={path}\n') + f.write(f"export NXF_SINGULARITY_CACHEDIR={path}\n") log.info("Successfully wrote to ~/.bashrc") except FileNotFoundError: log.error("Unable to find ~/.bashrc") @@ -199,18 +228,6 @@ def download_workflow(self): self.get_singularity_images() # If '--compress' flag was set, ask user what compression type to be used - if self.compress: - self.compress_type = questionary.select( - "Choose compression type:", - choices=[ - "none", - "tar.gz", - "tar.bz2", - "zip", - ], - ).ask() - if self.compress_type == "none": - self.compress_type = None # Compress into an archive if self.compress_type is not None: From 8a945e7039a84b93fca3f75e8062d3554b919685 Mon Sep 17 00:00:00 2001 From: Erik Danielsson Date: Fri, 16 Apr 2021 19:54:08 +0200 Subject: [PATCH 09/45] Clean up some unnecessary changes --- nf_core/__main__.py | 2 +- nf_core/download.py | 2 -- nf_core/launch.py | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index b0948fea7..272cf9708 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -205,7 +205,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") -@click.option("-r", "--release", help="Pipeline release") +@click.option("-r", "--release", type=str help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-c", diff --git a/nf_core/download.py b/nf_core/download.py index bddf0598b..6a9995e91 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -227,8 +227,6 @@ def download_workflow(self): self.find_container_images() self.get_singularity_images() - # If '--compress' flag was set, ask user what compression type to be used - # Compress into an archive if self.compress_type is not None: log.info("Compressing download..") diff --git a/nf_core/launch.py b/nf_core/launch.py index 5e304892d..78bca4aa2 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -16,7 +16,7 @@ import webbrowser import requests -import nf_core.schema, nf_core.utils, nf_core.download +import nf_core.schema, nf_core.utils log = logging.getLogger(__name__) @@ -197,11 +197,10 @@ def get_pipeline_schema(self): try: self.schema_obj.get_schema_path(self.pipeline, revision=self.pipeline_revision) self.schema_obj.load_lint_schema() - except AssertionError as a: + except AssertionError: # No schema found # Check that this was actually a pipeline if self.schema_obj.pipeline_dir is None or not os.path.exists(self.schema_obj.pipeline_dir): - log.info(f"dir: {a}") log.error("Could not find pipeline: {} ({})".format(self.pipeline, self.schema_obj.pipeline_dir)) return False if not os.path.exists(os.path.join(self.schema_obj.pipeline_dir, "nextflow.config")) and not os.path.exists( From ec7d48652a714f4b97388fe4f64aa0c47e943f62 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 16:17:39 +0200 Subject: [PATCH 10/45] Fix syntax error --- nf_core/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 31c36a7bf..615e013c8 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -205,7 +205,7 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all @nf_core_cli.command(help_priority=3) @click.argument("pipeline", required=False, metavar="") -@click.option("-r", "--release", type=str help="Pipeline release") +@click.option("-r", "--release", type=str, help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( "-c", From 115c11269712aca3d2b021ac59d4eb3a094a52d3 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 16:54:48 +0200 Subject: [PATCH 11/45] Testing and refactoring --- nf_core/download.py | 163 ++++++++++++++++++++++++-------------------- 1 file changed, 89 insertions(+), 74 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 6a9995e91..5f3291982 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -81,7 +81,7 @@ def __init__( outdir=None, compress_type=None, force=False, - container="none", + container=None, singularity_cache_only=False, parallel_downloads=4, ): @@ -90,19 +90,10 @@ def __init__( self.outdir = outdir self.output_filename = None self.compress_type = compress_type - if self.compress_type is None: - self.compress_type = self._confirm_compression() - if self.compress_type == "none": - self.compress_type = None - self.force = force - - if container is None: - container = self._confirm_container_download() + self.container = container self.singularity = container == "singularity" self.singularity_cache_only = singularity_cache_only - if self.singularity_cache_only is None and self.singularity: - self.singularity_cache_only = self._confirm_singularity_cache() self.parallel_downloads = parallel_downloads # Sanity checks @@ -116,6 +107,11 @@ def __init__( self.nf_config = dict() self.containers = list() + # Fetch remote workflows + self.wfs = nf_core.list.Workflows() + self.wfs.get_remote_workflows() + self.wf_branches = {} + def _confirm_compression(self): return questionary.select( "Choose compression type:", @@ -139,29 +135,12 @@ def _confirm_singularity_cache(self): def download_workflow(self): """Starts a nf-core workflow download.""" - # Fetch remote workflows - wfs = nf_core.list.Workflows() - wfs.get_remote_workflows() - - # Prompt user if pipeline name was not specified - if self.pipeline is None: - self.pipeline = questionary.autocomplete( - "Pipeline name:", - choices=[wf.name for wf in wfs.remote_workflows], - style=nf_core.utils.nfcore_question_style, - ).ask() - # Prompt user for release tag if '--release' was set - if self.release is None: - try: - release_tags = self.fetch_release_tags() - except LookupError: - sys.exit(1) - self.release = questionary.select("Select release:", choices=release_tags).ask() + self.prompt_inputs() # Get workflow details try: - self.fetch_workflow_details(wfs) + self.fetch_workflow_details() except LookupError: sys.exit(1) @@ -232,6 +211,41 @@ def download_workflow(self): log.info("Compressing download..") self.compress_download() + def prompt_inputs(self): + """Interactively prompt the user for any missing flags""" + + # Prompt user if pipeline name was not specified + if self.pipeline is None: + self.pipeline = questionary.autocomplete( + "Pipeline name:", + choices=[wf.name for wf in self.wfs.remote_workflows], + style=nf_core.utils.nfcore_question_style, + ).ask() + + # Prompt user for release tag if '--release' was not set + if self.release is None: + try: + release_tags = self.fetch_release_tags() + except LookupError: + sys.exit(1) + self.release = questionary.select("Select release / branch:", choices=release_tags).ask() + + # Download singularity container? + if self.container is None: + self.container = self._confirm_container_download() + + # Use $NXF_SINGULARITY_CACHEDIR ? + if self.singularity_cache_only is None and self.singularity: + self.singularity_cache_only = self._confirm_singularity_cache() + + # Compress the downloaded files? + if self.compress_type is None: + self.compress_type = self._confirm_compression() + + # Correct type for no-compression + if self.compress_type == "none": + self.compress_type = None + def fetch_release_tags(self): """Fetches tag names of pipeline releases from github @@ -241,67 +255,68 @@ def fetch_release_tags(self): Raises: LookupError, if no releases were found """ - # Fetch releases from github api - releases_url = "https://api.github.com/repos/nf-core/{}/releases".format(self.pipeline) - response = requests.get(releases_url) + + release_tags = [] + + # We get releases from https://nf-co.re/pipelines.json + for wf in self.wfs.remote_workflows: + if wf.full_name == self.pipeline or wf.name == self.pipeline: + if len(wf.releases) > 0: + releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) + release_tags = list(map(lambda release: release.get("tag_name", None), releases)) + + # Fetch branches from github api + branches_url = "https://api.github.com/repos/nf-core/{}/branches".format(self.pipeline) + branch_response = requests.get(branches_url) # Filter out the release tags and sort them - release_tags = map(lambda release: release.get("tag_name", None), response.json()) - release_tags = filter(lambda tag: tag != None, release_tags) - release_tags = list(release_tags) - if len(release_tags) == 0: - log.error("Unable to find any releases!") - raise LookupError - release_tags = sorted(release_tags, key=lambda tag: tuple(tag.split(".")), reverse=True) + for branch in branch_response.json(): + self.wf_branches[branch["name"]] = branch["commit"]["sha"] + release_tags.extend( + [ + b + for b in self.wf_branches.keys() + if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") + ] + ) + return release_tags - def fetch_workflow_details(self, wfs): + def fetch_workflow_details(self): """Fetches details of a nf-core workflow to download. - Args: - wfs (nf_core.list.Workflows): A nf_core.list.Workflows object - Raises: LockupError, if the pipeline can not be found. """ # Get workflow download details - for wf in wfs.remote_workflows: + for wf in self.wfs.remote_workflows: if wf.full_name == self.pipeline or wf.name == self.pipeline: # Set pipeline name self.wf_name = wf.name - # Find latest release hash - if self.release is None and len(wf.releases) > 0: - # Sort list of releases so that most recent is first - wf.releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - self.release = wf.releases[0]["tag_name"] - self.wf_sha = wf.releases[0]["tag_sha"] - log.debug("No release specified. Using latest release: {}".format(self.release)) - # Find specified release hash - elif self.release is not None: - for r in wf.releases: - if r["tag_name"] == self.release.lstrip("v"): - self.wf_sha = r["tag_sha"] - break + # Find specified release / branch hash + if self.release is not None: + + # Branch + if self.release in self.wf_branches.keys(): + self.wf_sha = self.wf_branches[self.release] + + # Release else: - log.error("Not able to find release '{}' for {}".format(self.release, wf.full_name)) - log.info( - "Available {} releases: {}".format( - wf.full_name, ", ".join([r["tag_name"] for r in wf.releases]) + for r in wf.releases: + if r["tag_name"] == self.release.lstrip("v"): + self.wf_sha = r["tag_sha"] + break + else: + log.error("Not able to find release '{}' for {}".format(self.release, wf.full_name)) + log.info( + "Available {} releases: {}".format( + wf.full_name, ", ".join([r["tag_name"] for r in wf.releases]) + ) ) - ) - raise LookupError("Not able to find release '{}' for {}".format(self.release, wf.full_name)) - - # Must be a dev-only pipeline - elif not self.release: - self.release = "dev" - self.wf_sha = "master" # Cheating a little, but GitHub download link works - log.warning( - "Pipeline is in development - downloading current code on master branch.\n" - + "This is likely to change soon should not be considered fully reproducible." - ) + raise LookupError("Not able to find release '{}' for {}".format(self.release, wf.full_name)) # Set outdir name if not defined if not self.outdir: @@ -330,7 +345,7 @@ def fetch_workflow_details(self, wfs): self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.release) else: log.error("Not able to find pipeline '{}'".format(self.pipeline)) - log.info("Available pipelines: {}".format(", ".join([w.name for w in wfs.remote_workflows]))) + log.info("Available pipelines: {}".format(", ".join([w.name for w in self.wfs.remote_workflows]))) raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) def download_wf_files(self): From 935b64e4cb8c0a4d4c477eafdb4d8a4d45629e74 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 17:01:11 +0200 Subject: [PATCH 12/45] Fix behaviour when pipeline name doesn't exist --- nf_core/download.py | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 5f3291982..32e5d26c8 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -224,11 +224,9 @@ def prompt_inputs(self): # Prompt user for release tag if '--release' was not set if self.release is None: - try: - release_tags = self.fetch_release_tags() - except LookupError: - sys.exit(1) - self.release = questionary.select("Select release / branch:", choices=release_tags).ask() + release_tags = self.fetch_release_tags() + if len(release_tags) > 0: + self.release = questionary.select("Select release / branch:", choices=release_tags).ask() # Download singularity container? if self.container is None: @@ -263,22 +261,26 @@ def fetch_release_tags(self): if wf.full_name == self.pipeline or wf.name == self.pipeline: if len(wf.releases) > 0: releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - release_tags = list(map(lambda release: release.get("tag_name", None), releases)) - - # Fetch branches from github api - branches_url = "https://api.github.com/repos/nf-core/{}/branches".format(self.pipeline) - branch_response = requests.get(branches_url) - - # Filter out the release tags and sort them - for branch in branch_response.json(): - self.wf_branches[branch["name"]] = branch["commit"]["sha"] - release_tags.extend( - [ - b - for b in self.wf_branches.keys() - if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") - ] - ) + release_tags = list(map(lambda release: release.get("tag_name"), releases)) + + try: + # Fetch branches from github api + branches_url = f"https://api.github.com/repos/nf-core/{self.pipeline}/branches" + branch_response = requests.get(branches_url) + + # Filter out the release tags and sort them + for branch in branch_response.json(): + self.wf_branches[branch["name"]] = branch["commit"]["sha"] + release_tags.extend( + [ + b + for b in self.wf_branches.keys() + if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") + ] + ) + except TypeError: + # This will be picked up later if not a repo, just log for now + log.debug("Couldn't fetch branches - invalid repo?") return release_tags @@ -345,7 +347,7 @@ def fetch_workflow_details(self): self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.release) else: log.error("Not able to find pipeline '{}'".format(self.pipeline)) - log.info("Available pipelines: {}".format(", ".join([w.name for w in self.wfs.remote_workflows]))) + log.info("Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows]))) raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) def download_wf_files(self): From b41337d5fc1d9829e5a66964db1f49e24da51832 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 17:07:15 +0200 Subject: [PATCH 13/45] Better cli styling --- nf_core/download.py | 17 +++++++++++++---- nf_core/launch.py | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 32e5d26c8..12f910636 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -121,6 +121,7 @@ def _confirm_compression(self): "tar.bz2", "zip", ], + style=nf_core.utils.nfcore_question_style, ).ask() def _confirm_container_download(self): @@ -226,7 +227,9 @@ def prompt_inputs(self): if self.release is None: release_tags = self.fetch_release_tags() if len(release_tags) > 0: - self.release = questionary.select("Select release / branch:", choices=release_tags).ask() + self.release = questionary.select( + "Select release / branch:", choices=release_tags, style=nf_core.utils.nfcore_question_style + ).ask() # Download singularity container? if self.container is None: @@ -347,7 +350,9 @@ def fetch_workflow_details(self): self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.release) else: log.error("Not able to find pipeline '{}'".format(self.pipeline)) - log.info("Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows]))) + log.info( + "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) + ) raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) def download_wf_files(self): @@ -755,7 +760,11 @@ def compress_download(self): with tarfile.open(self.output_filename, "w:{}".format(ctype)) as tar: tar.add(self.outdir, arcname=os.path.basename(self.outdir)) tar_flags = "xzf" if ctype == "gz" else "xjf" - log.info("Command to extract files: tar -{} {}".format(tar_flags, self.output_filename)) + log.info( + "Command to extract files: [bright_magenta on grey0] tar -{} {} [/]".format( + tar_flags, self.output_filename + ) + ) # .zip files if self.compress_type == "zip": @@ -796,7 +805,7 @@ def validate_md5(self, fname, expected=None): file_hash = hash_md5.hexdigest() if expected is None: - log.info("MD5 checksum for {}: {}".format(fname, file_hash)) + log.info("MD5 checksum for '{}': '{}'".format(fname, file_hash)) else: if file_hash == expected: log.debug("md5 sum of image matches expected: {}".format(expected)) diff --git a/nf_core/launch.py b/nf_core/launch.py index 78bca4aa2..8f131939d 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -184,7 +184,9 @@ def get_pipeline_schema(self): try: release_tags = self.try_fetch_release_tags() self.pipeline_revision = questionary.select( - "Please select a release:", choices=release_tags + "Please select a release:", + choices=release_tags, + style=nf_core.utils.nfcore_question_style, ).ask() except LookupError: pass From 48f9146ba5c7dd85279fe262e573a22e004a834b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 19:34:09 +0200 Subject: [PATCH 14/45] Fast fail for non-existant repos --- nf_core/download.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 12f910636..a7d10f963 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -137,12 +137,12 @@ def _confirm_singularity_cache(self): def download_workflow(self): """Starts a nf-core workflow download.""" - self.prompt_inputs() - # Get workflow details try: + self.prompt_inputs() self.fetch_workflow_details() - except LookupError: + except LookupError as e: + log.critical(e) sys.exit(1) summary_log = [ @@ -223,6 +223,26 @@ def prompt_inputs(self): style=nf_core.utils.nfcore_question_style, ).ask() + # Fast-fail for unrecognised pipelines (we check again at the end) + for wf in self.wfs.remote_workflows: + if wf.full_name == self.pipeline or wf.name == self.pipeline: + break + else: + # Non nf-core GitHub repo + if self.pipeline.count("/") == 1: + gh_response = requests.get(f"https://api.github.com/repos/{self.pipeline}") + try: + assert gh_response.json()["message"] == "Not Found" + except AssertionError: + pass + else: + raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + else: + log.info( + "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) + ) + raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + # Prompt user for release tag if '--release' was not set if self.release is None: release_tags = self.fetch_release_tags() From 0fc047713a2b4a82e8b229a9d50bf2f04aff32b9 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 19:37:33 +0200 Subject: [PATCH 15/45] More testing - bugfix --- nf_core/download.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index a7d10f963..496b0fbc0 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -191,7 +191,7 @@ def download_workflow(self): os.remove(self.output_filename) # Summary log - log.info("Saving {}\n {}".format(self.pipeline, "\n ".join(summary_log))) + log.info("Saving '{}'\n {}".format(self.pipeline, "\n ".join(summary_log))) # Download the pipeline files log.info("Downloading workflow files from GitHub") @@ -232,7 +232,7 @@ def prompt_inputs(self): if self.pipeline.count("/") == 1: gh_response = requests.get(f"https://api.github.com/repos/{self.pipeline}") try: - assert gh_response.json()["message"] == "Not Found" + assert gh_response.json().get("message") == "Not Found" except AssertionError: pass else: @@ -357,7 +357,7 @@ def fetch_workflow_details(self): if self.pipeline.count("/") == 1: # Looks like a GitHub address - try working with this repo log.warning("Pipeline name doesn't match any nf-core workflows") - log.info("Pipeline name looks like a GitHub address - attempting to download anyway") + log.info("Pipeline name looks like a GitHub address - attempting to download") self.wf_name = self.pipeline if not self.release: self.release = "master" From 59e06543e366ad0494b8d6ee61bada4a229b0909 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Wed, 28 Apr 2021 19:49:24 +0200 Subject: [PATCH 16/45] Clean up + finish refactor for --container instead of --singularity --- nf_core/__main__.py | 4 ++-- nf_core/download.py | 45 ++++++++++++++++++++------------------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 615e013c8..3fffbd24d 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -215,10 +215,10 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @click.option( - "-C", + "-i", "--container", type=click.Choice(["none", "singularity"]), - help="Download images", + help="Download software container images", ) @click.option( "-s", diff --git a/nf_core/download.py b/nf_core/download.py index 496b0fbc0..589b6c2ab 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -92,15 +92,9 @@ def __init__( self.compress_type = compress_type self.force = force self.container = container - self.singularity = container == "singularity" self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads - # Sanity checks - if self.singularity_cache_only and not self.singularity: - log.error("Command has '--singularity-cache' set, but '--container' is 'none'") - sys.exit(1) - self.wf_name = None self.wf_sha = None self.wf_download_url = None @@ -125,11 +119,14 @@ def _confirm_compression(self): ).ask() def _confirm_container_download(self): - should_download = Confirm.ask(f"Should singularity image be downloaded?") - if should_download: - return "singularity" - else: - return "none" + return questionary.select( + "Download software container images:", + choices=[ + "none", + "singularity", + ], + style=nf_core.utils.nfcore_question_style, + ).ask() def _confirm_singularity_cache(self): return Confirm.ask(f"Should singularity image be cached?") @@ -145,11 +142,8 @@ def download_workflow(self): log.critical(e) sys.exit(1) - summary_log = [ - "Pipeline release: '{}'".format(self.release), - "Pull singularity containers: '{}'".format("Yes" if self.singularity else "No"), - ] - if self.singularity: + summary_log = [f"Pipeline release: '{self.release}'", f"Pull containers: '{self.container}'"] + if self.container == "singularity": export_in_file = os.popen('grep -c "export NXF_SINGULARITY_CACHEDIR" ~/.bashrc').read().strip("\n") != "0" if not export_in_file: append_to_file = Confirm.ask("Add 'export NXF_SINGULARITY_CACHEDIR' to .bashrc?") @@ -203,7 +197,7 @@ def download_workflow(self): self.wf_use_local_configs() # Download the singularity images - if self.singularity: + if self.container == "singularity": self.find_container_images() self.get_singularity_images() @@ -256,9 +250,14 @@ def prompt_inputs(self): self.container = self._confirm_container_download() # Use $NXF_SINGULARITY_CACHEDIR ? - if self.singularity_cache_only is None and self.singularity: + if self.singularity_cache_only is None and self.container == "singularity": self.singularity_cache_only = self._confirm_singularity_cache() + # Sanity checks (for cli flags) + if self.singularity_cache_only and self.container != "singularity": + log.error("Command has '--singularity-cache' set, but '--container' is 'none'") + sys.exit(1) + # Compress the downloaded files? if self.compress_type is None: self.compress_type = self._confirm_compression() @@ -427,7 +426,7 @@ def wf_use_local_configs(self): nfconfig = nfconfig.replace(find_str, repl_str) # Append the singularity.cacheDir to the end if we need it - if self.singularity and not self.singularity_cache_only: + if self.container == "singularity" and not self.singularity_cache_only: nfconfig += ( f"\n\n// Added by `nf-core download` v{nf_core.__version__} //\n" + 'singularity.cacheDir = "${projectDir}/../singularity-images/"' @@ -780,11 +779,7 @@ def compress_download(self): with tarfile.open(self.output_filename, "w:{}".format(ctype)) as tar: tar.add(self.outdir, arcname=os.path.basename(self.outdir)) tar_flags = "xzf" if ctype == "gz" else "xjf" - log.info( - "Command to extract files: [bright_magenta on grey0] tar -{} {} [/]".format( - tar_flags, self.output_filename - ) - ) + log.info(f"Command to extract files: [bright_magenta]tar -{tar_flags} {self.output_filename}[/]") # .zip files if self.compress_type == "zip": @@ -825,7 +820,7 @@ def validate_md5(self, fname, expected=None): file_hash = hash_md5.hexdigest() if expected is None: - log.info("MD5 checksum for '{}': '{}'".format(fname, file_hash)) + log.info("MD5 checksum for '{}': [blue]{}[/]".format(fname, file_hash)) else: if file_hash == expected: log.debug("md5 sum of image matches expected: {}".format(expected)) From bb6b6d2a16ab084f6a9a45e470a23a9c2eb4982f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 29 Apr 2021 23:36:30 +0200 Subject: [PATCH 17/45] Fix Singularity installation check --- nf_core/download.py | 65 ++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 589b6c2ab..03582c1d9 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -129,7 +129,7 @@ def _confirm_container_download(self): ).ask() def _confirm_singularity_cache(self): - return Confirm.ask(f"Should singularity image be cached?") + return Confirm.ask(f"[blue bold]?[/] [white bold]Should singularity image be cached?[/]") def download_workflow(self): """Starts a nf-core workflow download.""" @@ -199,7 +199,11 @@ def download_workflow(self): # Download the singularity images if self.container == "singularity": self.find_container_images() - self.get_singularity_images() + try: + self.get_singularity_images() + except OSError as e: + log.critical(f"[red]{e}[/]") + sys.exit(1) # Compress into an archive if self.compress_type is not None: @@ -447,7 +451,7 @@ def find_container_images(self): `nextflow config` at the time of writing, so we scrape the pipeline files. """ - log.info("Fetching container names for workflow") + log.debug("Fetching container names for workflow") # Use linting code to parse the pipeline nextflow config self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, "workflow")) @@ -490,11 +494,6 @@ def get_singularity_images(self): if len(self.containers) == 0: log.info("No container names found in workflow") else: - if not os.environ.get("NXF_SINGULARITY_CACHEDIR"): - log.info( - "[magenta]Tip: Set env var $NXF_SINGULARITY_CACHEDIR to use a central cache for container downloads" - ) - with DownloadProgress() as progress: task = progress.add_task("all_containers", total=len(self.containers), progress_type="summary") @@ -537,6 +536,10 @@ def get_singularity_images(self): # Pull using singularity containers_pull.append([container, out_path, cache_path]) + # Exit if we need to pull images and Singularity is not installed + if len(containers_pull) > 0 and shutil.which("singularity") is None: + raise OSError("Images need to be pulled from Docker, but Singularity is not installed") + # Go through each method of fetching containers in order for container in containers_exist: progress.update(task, description="Image file exists") @@ -739,35 +742,25 @@ def singularity_pull_image(self, container, out_path, cache_path, progress): # Progress bar to show that something is happening task = progress.add_task(container, start=False, total=False, progress_type="singularity_pull", current_log="") - # Try to use singularity to pull image - try: - # Run the singularity pull command - proc = subprocess.Popen( - singularity_command, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - bufsize=1, - ) - for line in proc.stdout: - log.debug(line.strip()) - progress.update(task, current_log=line.strip()) - - # Copy cached download if we are using the cache - if cache_path: - log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) - progress.update(task, current_log="Copying from cache to target directory") - shutil.copyfile(cache_path, out_path) - - progress.remove_task(task) + # Run the singularity pull command + proc = subprocess.Popen( + singularity_command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + bufsize=1, + ) + for line in proc.stdout: + log.debug(line.strip()) + progress.update(task, current_log=line.strip()) + + # Copy cached download if we are using the cache + if cache_path: + log.debug("Copying {} from cache: '{}'".format(container, os.path.basename(out_path))) + progress.update(task, current_log="Copying from cache to target directory") + shutil.copyfile(cache_path, out_path) - except OSError as e: - if e.errno == errno.ENOENT: - # Singularity is not installed - log.error("Singularity is not installed!") - else: - # Something else went wrong with singularity command - raise e + progress.remove_task(task) def compress_download(self): """Take the downloaded files and make a compressed .tar.gz archive.""" From d9c72074d139171b939e90d54c26ce73a4d5fc9d Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 00:19:19 +0200 Subject: [PATCH 18/45] Fix / rewrite code for singularity cachedir prompts + bashrc addition --- nf_core/download.py | 72 +++++++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 03582c1d9..9c97d4d38 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -128,9 +128,6 @@ def _confirm_container_download(self): style=nf_core.utils.nfcore_question_style, ).ask() - def _confirm_singularity_cache(self): - return Confirm.ask(f"[blue bold]?[/] [white bold]Should singularity image be cached?[/]") - def download_workflow(self): """Starts a nf-core workflow download.""" @@ -144,21 +141,9 @@ def download_workflow(self): summary_log = [f"Pipeline release: '{self.release}'", f"Pull containers: '{self.container}'"] if self.container == "singularity": - export_in_file = os.popen('grep -c "export NXF_SINGULARITY_CACHEDIR" ~/.bashrc').read().strip("\n") != "0" - if not export_in_file: - append_to_file = Confirm.ask("Add 'export NXF_SINGULARITY_CACHEDIR' to .bashrc?") - if append_to_file: - path = Prompt.ask("Specify the path: ") - try: - with open(os.path.expanduser("~/.bashrc"), "a") as f: - f.write(f"export NXF_SINGULARITY_CACHEDIR={path}\n") - log.info("Successfully wrote to ~/.bashrc") - except FileNotFoundError: - log.error("Unable to find ~/.bashrc") - sys.exit(1) if os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: summary_log.append( - "Using '$NXF_SINGULARITY_CACHEDIR': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) + "Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) ) # Set an output filename now that we have the outdir @@ -254,12 +239,22 @@ def prompt_inputs(self): self.container = self._confirm_container_download() # Use $NXF_SINGULARITY_CACHEDIR ? - if self.singularity_cache_only is None and self.container == "singularity": - self.singularity_cache_only = self._confirm_singularity_cache() + if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None: + self.set_nxf_singularity_cachedir() + + # Use *only* $NXF_SINGULARITY_CACHEDIR without copying into target? + if ( + self.singularity_cache_only is None + and self.container == "singularity" + and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None + ): + self.singularity_cache_only = Confirm.ask( + f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the download folder?[/]" + ) # Sanity checks (for cli flags) if self.singularity_cache_only and self.container != "singularity": - log.error("Command has '--singularity-cache' set, but '--container' is 'none'") + log.error("Command has '--singularity-cache' set, but '--container' is not 'singularity'") sys.exit(1) # Compress the downloaded files? @@ -270,6 +265,45 @@ def prompt_inputs(self): if self.compress_type == "none": self.compress_type = None + def set_nxf_singularity_cachedir(self): + """Ask if the user wants to set a Singularity cache""" + + if Confirm.ask( + f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" + ): + cachedir_path = None + while cachedir_path is None: + cachedir_path = os.path.abspath( + Prompt.ask("[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)") + ) + if cachedir_path == "": + cachedir_path = False + elif not os.path.isdir(cachedir_path): + log.error(f"'{cachedir_path}' is not a directory.") + cachedir_path = None + if cachedir_path: + os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path + + # Ask if user wants this set in their .bashrc + bashrc_path = os.path.expanduser("~/.bashrc") + if not os.path.isfile(bashrc_path): + bashrc_path = os.path.expanduser("~/.bash_profile") + if not os.path.isfile(bashrc_path): + bashrc_path = False + if bashrc_path: + append_to_file = Confirm.ask( + f"[blue bold]?[/] [white bold]Add [green not bold]'export NXF_SINGULARITY_CACHEDIR=\"{cachedir_path}\"'[/] to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + ) + if append_to_file: + with open(os.path.expanduser(bashrc_path), "a") as f: + f.write( + "\n\n#######################################\n" + f"## Added by `nf-core download` v{nf_core.__version__} ##\n" + + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' + + "\n#######################################\n" + ) + log.info(f"Successfully wrote to {bashrc_path}") + def fetch_release_tags(self): """Fetches tag names of pipeline releases from github From 1c921d680f9a863887712b437325d3a8a1d853b1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 00:31:42 +0200 Subject: [PATCH 19/45] Restructure and reorganise prompts code --- nf_core/download.py | 220 +++++++++++++++++++++----------------------- 1 file changed, 104 insertions(+), 116 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 9c97d4d38..406652256 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -106,34 +106,17 @@ def __init__( self.wfs.get_remote_workflows() self.wf_branches = {} - def _confirm_compression(self): - return questionary.select( - "Choose compression type:", - choices=[ - "none", - "tar.gz", - "tar.bz2", - "zip", - ], - style=nf_core.utils.nfcore_question_style, - ).ask() - - def _confirm_container_download(self): - return questionary.select( - "Download software container images:", - choices=[ - "none", - "singularity", - ], - style=nf_core.utils.nfcore_question_style, - ).ask() - def download_workflow(self): """Starts a nf-core workflow download.""" # Get workflow details try: - self.prompt_inputs() + self.prompt_pipeline_name() + self.prompt_release() + self.prompt_container_download() + self.prompt_use_singularity_cachedir() + self.prompt_singularity_cachedir_only() + self.prompt_compression_type() self.fetch_workflow_details() except LookupError as e: log.critical(e) @@ -195,10 +178,9 @@ def download_workflow(self): log.info("Compressing download..") self.compress_download() - def prompt_inputs(self): - """Interactively prompt the user for any missing flags""" + def prompt_pipeline_name(self): + """Prompt for the pipeline name if not set with a flag""" - # Prompt user if pipeline name was not specified if self.pipeline is None: self.pipeline = questionary.autocomplete( "Pipeline name:", @@ -226,23 +208,99 @@ def prompt_inputs(self): ) raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + def prompt_release(self): + """Prompt for pipeline release / branch""" # Prompt user for release tag if '--release' was not set if self.release is None: - release_tags = self.fetch_release_tags() + release_tags = [] + + # We get releases from https://nf-co.re/pipelines.json + for wf in self.wfs.remote_workflows: + if wf.full_name == self.pipeline or wf.name == self.pipeline: + if len(wf.releases) > 0: + releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) + release_tags = list(map(lambda release: release.get("tag_name"), releases)) + + try: + # Fetch branches from github api + branches_url = f"https://api.github.com/repos/nf-core/{self.pipeline}/branches" + branch_response = requests.get(branches_url) + + # Filter out the release tags and sort them + for branch in branch_response.json(): + self.wf_branches[branch["name"]] = branch["commit"]["sha"] + release_tags.extend( + [ + b + for b in self.wf_branches.keys() + if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") + ] + ) + except TypeError: + # This will be picked up later if not a repo, just log for now + log.debug("Couldn't fetch branches - invalid repo?") + if len(release_tags) > 0: self.release = questionary.select( "Select release / branch:", choices=release_tags, style=nf_core.utils.nfcore_question_style ).ask() - # Download singularity container? + def prompt_container_download(self): + """Prompt whether to download container images or not""" + if self.container is None: - self.container = self._confirm_container_download() + self.container = questionary.select( + "Download software container images:", + choices=[ + "none", + "singularity", + ], + style=nf_core.utils.nfcore_question_style, + ).ask() - # Use $NXF_SINGULARITY_CACHEDIR ? + def prompt_use_singularity_cachedir(self): + """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None: - self.set_nxf_singularity_cachedir() + if Confirm.ask( + f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" + ): + # Prompt user for a cache directory path + cachedir_path = None + while cachedir_path is None: + cachedir_path = os.path.abspath( + Prompt.ask("[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)") + ) + if cachedir_path == os.path.abspath(""): + log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") + cachedir_path = False + elif not os.path.isdir(cachedir_path): + log.error(f"'{cachedir_path}' is not a directory.") + cachedir_path = None + if cachedir_path: + os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path + + # Ask if user wants this set in their .bashrc + bashrc_path = os.path.expanduser("~/.bashrc") + if not os.path.isfile(bashrc_path): + bashrc_path = os.path.expanduser("~/.bash_profile") + if not os.path.isfile(bashrc_path): + bashrc_path = False + if bashrc_path: + append_to_file = Confirm.ask( + f"[blue bold]?[/] [white bold]Add [green not bold]'export NXF_SINGULARITY_CACHEDIR=\"{cachedir_path}\"'[/] to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + ) + if append_to_file: + with open(os.path.expanduser(bashrc_path), "a") as f: + f.write( + "\n\n#######################################\n" + f"## Added by `nf-core download` v{nf_core.__version__} ##\n" + + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' + + "\n#######################################\n" + ) + log.info(f"Successfully wrote to {bashrc_path}") - # Use *only* $NXF_SINGULARITY_CACHEDIR without copying into target? + def prompt_singularity_cachedir_only(self): + """Ask if we should *only* use $NXF_SINGULARITY_CACHEDIR without copying into target""" if ( self.singularity_cache_only is None and self.container == "singularity" @@ -252,98 +310,28 @@ def prompt_inputs(self): f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the download folder?[/]" ) - # Sanity checks (for cli flags) + # Sanity check if self.singularity_cache_only and self.container != "singularity": - log.error("Command has '--singularity-cache' set, but '--container' is not 'singularity'") - sys.exit(1) + raise LookupError("Command has '--singularity-cache' set, but '--container' is not 'singularity'") - # Compress the downloaded files? + def prompt_compression_type(self): + """Ask user if we should compress the downloaded files""" if self.compress_type is None: - self.compress_type = self._confirm_compression() + self.compress_type = questionary.select( + "Choose compression type:", + choices=[ + "none", + "tar.gz", + "tar.bz2", + "zip", + ], + style=nf_core.utils.nfcore_question_style, + ).ask() # Correct type for no-compression if self.compress_type == "none": self.compress_type = None - def set_nxf_singularity_cachedir(self): - """Ask if the user wants to set a Singularity cache""" - - if Confirm.ask( - f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" - ): - cachedir_path = None - while cachedir_path is None: - cachedir_path = os.path.abspath( - Prompt.ask("[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)") - ) - if cachedir_path == "": - cachedir_path = False - elif not os.path.isdir(cachedir_path): - log.error(f"'{cachedir_path}' is not a directory.") - cachedir_path = None - if cachedir_path: - os.environ["NXF_SINGULARITY_CACHEDIR"] = cachedir_path - - # Ask if user wants this set in their .bashrc - bashrc_path = os.path.expanduser("~/.bashrc") - if not os.path.isfile(bashrc_path): - bashrc_path = os.path.expanduser("~/.bash_profile") - if not os.path.isfile(bashrc_path): - bashrc_path = False - if bashrc_path: - append_to_file = Confirm.ask( - f"[blue bold]?[/] [white bold]Add [green not bold]'export NXF_SINGULARITY_CACHEDIR=\"{cachedir_path}\"'[/] to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" - ) - if append_to_file: - with open(os.path.expanduser(bashrc_path), "a") as f: - f.write( - "\n\n#######################################\n" - f"## Added by `nf-core download` v{nf_core.__version__} ##\n" - + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' - + "\n#######################################\n" - ) - log.info(f"Successfully wrote to {bashrc_path}") - - def fetch_release_tags(self): - """Fetches tag names of pipeline releases from github - - Returns: - release_tags (list[str]): Returns list of release tags - - Raises: - LookupError, if no releases were found - """ - - release_tags = [] - - # We get releases from https://nf-co.re/pipelines.json - for wf in self.wfs.remote_workflows: - if wf.full_name == self.pipeline or wf.name == self.pipeline: - if len(wf.releases) > 0: - releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - release_tags = list(map(lambda release: release.get("tag_name"), releases)) - - try: - # Fetch branches from github api - branches_url = f"https://api.github.com/repos/nf-core/{self.pipeline}/branches" - branch_response = requests.get(branches_url) - - # Filter out the release tags and sort them - for branch in branch_response.json(): - self.wf_branches[branch["name"]] = branch["commit"]["sha"] - release_tags.extend( - [ - b - for b in self.wf_branches.keys() - if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") - ] - ) - except TypeError: - # This will be picked up later if not a repo, just log for now - log.debug("Couldn't fetch branches - invalid repo?") - - return release_tags - def fetch_workflow_details(self): """Fetches details of a nf-core workflow to download. From 9f0fe810fde0b2e5bf956fc745e262b316810947 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 00:39:48 +0200 Subject: [PATCH 20/45] Tweaks to log messages --- nf_core/download.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 406652256..f7239092b 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -123,11 +123,10 @@ def download_workflow(self): sys.exit(1) summary_log = [f"Pipeline release: '{self.release}'", f"Pull containers: '{self.container}'"] - if self.container == "singularity": - if os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: - summary_log.append( - "Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) - ) + if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None: + summary_log.append( + "Using [blue]$NXF_SINGULARITY_CACHEDIR[/]': {}".format(os.environ["NXF_SINGULARITY_CACHEDIR"]) + ) # Set an output filename now that we have the outdir if self.compress_type is not None: @@ -297,7 +296,10 @@ def prompt_use_singularity_cachedir(self): + f'export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"' + "\n#######################################\n" ) - log.info(f"Successfully wrote to {bashrc_path}") + log.info(f"Successfully wrote to [blue]{bashrc_path}[/]") + log.warning( + "You will need reload your terminal after the download completes for this to take effect." + ) def prompt_singularity_cachedir_only(self): """Ask if we should *only* use $NXF_SINGULARITY_CACHEDIR without copying into target""" @@ -381,8 +383,7 @@ def fetch_workflow_details(self): # If we got this far, must not be a nf-core pipeline if self.pipeline.count("/") == 1: # Looks like a GitHub address - try working with this repo - log.warning("Pipeline name doesn't match any nf-core workflows") - log.info("Pipeline name looks like a GitHub address - attempting to download") + log.debug("Pipeline name looks like a GitHub address - attempting to download") self.wf_name = self.pipeline if not self.release: self.release = "master" @@ -560,7 +561,7 @@ def get_singularity_images(self): # Exit if we need to pull images and Singularity is not installed if len(containers_pull) > 0 and shutil.which("singularity") is None: - raise OSError("Images need to be pulled from Docker, but Singularity is not installed") + raise OSError("Singularity is needed to pull images, but it is not installed") # Go through each method of fetching containers in order for container in containers_exist: @@ -806,10 +807,10 @@ def compress_download(self): filePath = os.path.join(folderName, filename) # Add file to zip zipObj.write(filePath) - log.info("Command to extract files: unzip {}".format(self.output_filename)) + log.info(f"Command to extract files: [bright_magenta]unzip {self.output_filename}[/]") # Delete original files - log.debug("Deleting uncompressed files: {}".format(self.outdir)) + log.debug(f"Deleting uncompressed files: '{self.outdir}'") shutil.rmtree(self.outdir) # Caclualte md5sum for output file From 24adc00117f9df01eb050758e215334a8df23a25 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 00:48:17 +0200 Subject: [PATCH 21/45] Use new rich console stderr argument, streamline imports --- nf_core/__main__.py | 6 +++--- nf_core/bump_version.py | 3 +-- nf_core/download.py | 29 ++++++++++++++--------------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 3fffbd24d..d4a8daf38 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -36,7 +36,7 @@ def run_nf_core(): rich.traceback.install(width=200, word_wrap=True, extra_lines=1) # Print nf-core header to STDERR - stderr = rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors()) + stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) stderr.print("\n[green]{},--.[grey39]/[green],-.".format(" " * 42), highlight=False) stderr.print("[blue] ___ __ __ __ ___ [green]/,-._.--~\\", highlight=False) stderr.print("[blue] |\ | |__ __ / ` / \ |__) |__ [yellow] } {", highlight=False) @@ -116,7 +116,7 @@ def nf_core_cli(verbose, log_file): log.addHandler( rich.logging.RichHandler( level=logging.DEBUG if verbose else logging.INFO, - console=rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors()), + console=rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()), show_time=False, markup=True, ) @@ -263,7 +263,7 @@ def licences(pipeline, json): # nf-core create def validate_wf_name_prompt(ctx, opts, value): - """ Force the workflow name to meet the nf-core requirements """ + """Force the workflow name to meet the nf-core requirements""" if not re.match(r"^[a-z]+$", value): click.echo("Invalid workflow name: must be lowercase without punctuation.") value = click.prompt(opts.prompt) diff --git a/nf_core/bump_version.py b/nf_core/bump_version.py index b770cb2e6..7788c9b55 100644 --- a/nf_core/bump_version.py +++ b/nf_core/bump_version.py @@ -3,7 +3,6 @@ a nf-core pipeline. """ -import click import logging import os import re @@ -12,7 +11,7 @@ import nf_core.utils log = logging.getLogger(__name__) -stderr = rich.console.Console(file=sys.stderr, force_terminal=nf_core.utils.rich_force_colors()) +stderr = rich.console.Console(stderr=True, force_terminal=nf_core.utils.rich_force_colors()) def bump_pipeline_version(pipeline_obj, new_version): diff --git a/nf_core/download.py b/nf_core/download.py index f7239092b..a32df4363 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -3,7 +3,6 @@ from __future__ import print_function -import errno from io import BytesIO import logging import hashlib @@ -17,18 +16,16 @@ import sys import tarfile import concurrent.futures -from rich.progress import BarColumn, DownloadColumn, TransferSpeedColumn, Progress -from rich.prompt import Confirm, Prompt +import rich +import rich.progress from zipfile import ZipFile import nf_core -import nf_core.list -import nf_core.utils log = logging.getLogger(__name__) -class DownloadProgress(Progress): +class DownloadProgress(rich.progress.Progress): """Custom Progress bar class, allowing us to have two progress bars with different columns / layouts. """ @@ -38,7 +35,7 @@ def get_renderables(self): if task.fields.get("progress_type") == "summary": self.columns = ( "[magenta]{task.description}", - BarColumn(bar_width=None), + rich.progress.BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.0f}%", "•", "[green]{task.completed}/{task.total} completed", @@ -46,18 +43,18 @@ def get_renderables(self): if task.fields.get("progress_type") == "download": self.columns = ( "[blue]{task.description}", - BarColumn(bar_width=None), + rich.progress.BarColumn(bar_width=None), "[progress.percentage]{task.percentage:>3.1f}%", "•", - DownloadColumn(), + rich.progress.DownloadColumn(), "•", - TransferSpeedColumn(), + rich.progress.TransferSpeedColumn(), ) if task.fields.get("progress_type") == "singularity_pull": self.columns = ( "[magenta]{task.description}", "[blue]{task.fields[current_log]}", - BarColumn(bar_width=None), + rich.progress.BarColumn(bar_width=None), ) yield self.make_tasks_table([task]) @@ -260,14 +257,16 @@ def prompt_container_download(self): def prompt_use_singularity_cachedir(self): """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None: - if Confirm.ask( + if rich.prompt.Confirm.ask( f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): # Prompt user for a cache directory path cachedir_path = None while cachedir_path is None: cachedir_path = os.path.abspath( - Prompt.ask("[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)") + rich.prompt.Prompt.ask( + "[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)" + ) ) if cachedir_path == os.path.abspath(""): log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") @@ -285,7 +284,7 @@ def prompt_use_singularity_cachedir(self): if not os.path.isfile(bashrc_path): bashrc_path = False if bashrc_path: - append_to_file = Confirm.ask( + append_to_file = rich.prompt.Confirm.ask( f"[blue bold]?[/] [white bold]Add [green not bold]'export NXF_SINGULARITY_CACHEDIR=\"{cachedir_path}\"'[/] to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" ) if append_to_file: @@ -308,7 +307,7 @@ def prompt_singularity_cachedir_only(self): and self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None ): - self.singularity_cache_only = Confirm.ask( + self.singularity_cache_only = rich.prompt.Confirm.ask( f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the download folder?[/]" ) From fe77d55b8086ea2f7443f0c614252fc316c41b02 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 01:01:35 +0200 Subject: [PATCH 22/45] Unsafe ask, colour select for releases / branches --- nf_core/download.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index a32df4363..7bd8b68bb 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -23,6 +23,7 @@ import nf_core log = logging.getLogger(__name__) +stderr = rich.console.Console(stderr=True, highlight=False, force_terminal=nf_core.utils.rich_force_colors()) class DownloadProgress(rich.progress.Progress): @@ -178,11 +179,12 @@ def prompt_pipeline_name(self): """Prompt for the pipeline name if not set with a flag""" if self.pipeline is None: + stderr.print("Specify the name of a nf-core pipeline or a GitHub repository name (user/repo).") self.pipeline = questionary.autocomplete( "Pipeline name:", choices=[wf.name for wf in self.wfs.remote_workflows], style=nf_core.utils.nfcore_question_style, - ).ask() + ).unsafe_ask() # Fast-fail for unrecognised pipelines (we check again at the end) for wf in self.wfs.remote_workflows: @@ -208,14 +210,16 @@ def prompt_release(self): """Prompt for pipeline release / branch""" # Prompt user for release tag if '--release' was not set if self.release is None: - release_tags = [] + choices = [] # We get releases from https://nf-co.re/pipelines.json for wf in self.wfs.remote_workflows: if wf.full_name == self.pipeline or wf.name == self.pipeline: if len(wf.releases) > 0: releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - release_tags = list(map(lambda release: release.get("tag_name"), releases)) + for tag in map(lambda release: release.get("tag_name"), releases): + tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] + choices.append(questionary.Choice(title=tag_display, value=tag)) try: # Fetch branches from github api @@ -225,21 +229,25 @@ def prompt_release(self): # Filter out the release tags and sort them for branch in branch_response.json(): self.wf_branches[branch["name"]] = branch["commit"]["sha"] - release_tags.extend( - [ - b - for b in self.wf_branches.keys() - if b != "TEMPLATE" and b != "initial_commit" and not b.startswith("nf-core-template-merge") - ] - ) + + for branch in self.wf_branches.keys(): + if ( + branch != "TEMPLATE" + and branch != "initial_commit" + and not branch.startswith("nf-core-template-merge") + ): + branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] + choices.append(questionary.Choice(title=branch_display, value=branch)) + except TypeError: # This will be picked up later if not a repo, just log for now log.debug("Couldn't fetch branches - invalid repo?") - if len(release_tags) > 0: + if len(choices) > 0: + stderr.print("\nChoose the release or branch that should be downloaded.") self.release = questionary.select( - "Select release / branch:", choices=release_tags, style=nf_core.utils.nfcore_question_style - ).ask() + "Select release / branch:", choices=choices, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -252,7 +260,7 @@ def prompt_container_download(self): "singularity", ], style=nf_core.utils.nfcore_question_style, - ).ask() + ).unsafe_ask() def prompt_use_singularity_cachedir(self): """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" @@ -327,7 +335,7 @@ def prompt_compression_type(self): "zip", ], style=nf_core.utils.nfcore_question_style, - ).ask() + ).unsafe_ask() # Correct type for no-compression if self.compress_type == "none": From 3c8006784833ed2a9c9c6155934b318d00f7e166 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 01:16:21 +0200 Subject: [PATCH 23/45] Write some help text for prompts --- nf_core/download.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 7bd8b68bb..956018e26 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -23,7 +23,9 @@ import nf_core log = logging.getLogger(__name__) -stderr = rich.console.Console(stderr=True, highlight=False, force_terminal=nf_core.utils.rich_force_colors()) +stderr = rich.console.Console( + stderr=True, style="dim", highlight=False, force_terminal=nf_core.utils.rich_force_colors() +) class DownloadProgress(rich.progress.Progress): @@ -253,6 +255,7 @@ def prompt_container_download(self): """Prompt whether to download container images or not""" if self.container is None: + stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container = questionary.select( "Download software container images:", choices=[ @@ -265,6 +268,10 @@ def prompt_container_download(self): def prompt_use_singularity_cachedir(self): """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None: + stderr.print( + "\nNextflow and nf-core can use an environment variable called [blue]$NXF_SINGULARITY_CACHEDIR[/] that is a path to a directory where remote Singularity images are stored. " + "This allows downloaded images to be cached in a central location." + ) if rich.prompt.Confirm.ask( f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): @@ -292,8 +299,13 @@ def prompt_use_singularity_cachedir(self): if not os.path.isfile(bashrc_path): bashrc_path = False if bashrc_path: + stderr.print( + f"\nSo that [blue]$NXF_SINGULARITY_CACHEDIR[/] is always defined, you can add it to your [blue not bold]~/{os.path.basename(bashrc_path)}[/] file ." + "This will then be autmoatically set every time you open a new terminal. We can add the following line to this file for you: \n" + f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]' + ) append_to_file = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [white bold]Add [green not bold]'export NXF_SINGULARITY_CACHEDIR=\"{cachedir_path}\"'[/] to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + f"[blue bold]?[/] [white bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" ) if append_to_file: with open(os.path.expanduser(bashrc_path), "a") as f: @@ -315,8 +327,13 @@ def prompt_singularity_cachedir_only(self): and self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None ): + stderr.print( + "\nIf you are working on the same system where you will run Nextflow, you can leave the downloaded images in the " + "[blue not bold]$NXF_SINGULARITY_CACHEDIR[/] folder, Nextflow will automatically find them. " + "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." + ) self.singularity_cache_only = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the download folder?[/]" + f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" ) # Sanity check @@ -326,6 +343,13 @@ def prompt_singularity_cachedir_only(self): def prompt_compression_type(self): """Ask user if we should compress the downloaded files""" if self.compress_type is None: + stderr.print( + "\nIf transferring the downloaded files to another system, it can be convenient to have everything compressed in a single file." + ) + if self.container == "singularity": + stderr.print( + "[bold]This is [italic]not[/] recommended when downloading Singularity images, as it can take a long time and saves very little space." + ) self.compress_type = questionary.select( "Choose compression type:", choices=[ From f9c9fdc13cd3843242360a8b0ae5d0092dac4f38 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 02:04:04 +0200 Subject: [PATCH 24/45] Refactored code that gets branches and releases and hashes. Rewired some cli options. --- nf_core/__main__.py | 7 +- nf_core/download.py | 163 +++++++++++++++++++++-------------------- tests/test_download.py | 6 +- 3 files changed, 92 insertions(+), 84 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index d4a8daf38..9a56bad95 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -221,10 +221,9 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all help="Download software container images", ) @click.option( - "-s", - "--singularity-cache", - type=click.Choice(["yes", "no"]), - help="Don't copy images to the output directory, don't set 'singularity.cacheDir' in workflow", + "-s/-x", + "--singularity-cache/--no-singularity-cache", + help="Do / don't copy images to the output directory and set 'singularity.cacheDir' in workflow", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") def download(pipeline, release, outdir, compress, force, container, singularity_cache, parallel_downloads): diff --git a/nf_core/download.py b/nf_core/download.py index 956018e26..83465f0c8 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -21,6 +21,8 @@ from zipfile import ZipFile import nf_core +import nf_core.list +import nf_core.utils log = logging.getLogger(__name__) stderr = rich.console.Console( @@ -104,6 +106,7 @@ def __init__( # Fetch remote workflows self.wfs = nf_core.list.Workflows() self.wfs.get_remote_workflows() + self.wf_releases = {} self.wf_branches = {} def download_workflow(self): @@ -112,12 +115,13 @@ def download_workflow(self): # Get workflow details try: self.prompt_pipeline_name() + self.fetch_workflow_details() self.prompt_release() + self.get_release_hash() self.prompt_container_download() self.prompt_use_singularity_cachedir() self.prompt_singularity_cachedir_only() self.prompt_compression_type() - self.fetch_workflow_details() except LookupError as e: log.critical(e) sys.exit(1) @@ -214,36 +218,16 @@ def prompt_release(self): if self.release is None: choices = [] - # We get releases from https://nf-co.re/pipelines.json - for wf in self.wfs.remote_workflows: - if wf.full_name == self.pipeline or wf.name == self.pipeline: - if len(wf.releases) > 0: - releases = sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - for tag in map(lambda release: release.get("tag_name"), releases): - tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] - choices.append(questionary.Choice(title=tag_display, value=tag)) + # Releases + if len(self.wf_releases) > 0: + for tag in map(lambda release: release.get("tag_name"), self.wf_releases): + tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] + choices.append(questionary.Choice(title=tag_display, value=tag)) - try: - # Fetch branches from github api - branches_url = f"https://api.github.com/repos/nf-core/{self.pipeline}/branches" - branch_response = requests.get(branches_url) - - # Filter out the release tags and sort them - for branch in branch_response.json(): - self.wf_branches[branch["name"]] = branch["commit"]["sha"] - - for branch in self.wf_branches.keys(): - if ( - branch != "TEMPLATE" - and branch != "initial_commit" - and not branch.startswith("nf-core-template-merge") - ): - branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] - choices.append(questionary.Choice(title=branch_display, value=branch)) - - except TypeError: - # This will be picked up later if not a repo, just log for now - log.debug("Couldn't fetch branches - invalid repo?") + # Branches + for branch in self.wf_branches.keys(): + branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] + choices.append(questionary.Choice(title=branch_display, value=branch)) if len(choices) > 0: stderr.print("\nChoose the release or branch that should be downloaded.") @@ -251,6 +235,38 @@ def prompt_release(self): "Select release / branch:", choices=choices, style=nf_core.utils.nfcore_question_style ).unsafe_ask() + def get_release_hash(self): + """Find specified release / branch hash""" + + # Branch + if self.release in self.wf_branches.keys(): + self.wf_sha = self.wf_branches[self.release] + + # Release + else: + for r in self.wf_releases: + if r["tag_name"] == self.release.lstrip("v"): + self.wf_sha = r["tag_sha"] + break + + # Can't find the release or branch - throw an error + else: + log.error("Not able to find release '{}' for {}".format(self.release, self.wf_name)) + log.info( + "Available {} releases: {}".format( + self.wf_name, ", ".join([r["tag_name"] for r in self.wf_releases]) + ) + ) + log.info("Available {} branches: '{}'".format(self.wf_name, "', '".join(self.wf_branches.keys()))) + raise LookupError("Not able to find release / branch '{}' for {}".format(self.release, self.wf_name)) + + # Set the outdir + if not self.outdir: + self.outdir = "{}-{}".format(self.wf_name.replace("/", "-").lower(), self.release) + + # Set the download URL and return + self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.wf_name, self.wf_sha) + def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -379,58 +395,47 @@ def fetch_workflow_details(self): # Set pipeline name self.wf_name = wf.name - # Find specified release / branch hash - if self.release is not None: + # Store releases + self.wf_releases = list( + sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) + ) - # Branch - if self.release in self.wf_branches.keys(): - self.wf_sha = self.wf_branches[self.release] + break - # Release - else: - for r in wf.releases: - if r["tag_name"] == self.release.lstrip("v"): - self.wf_sha = r["tag_sha"] - break - else: - log.error("Not able to find release '{}' for {}".format(self.release, wf.full_name)) - log.info( - "Available {} releases: {}".format( - wf.full_name, ", ".join([r["tag_name"] for r in wf.releases]) - ) - ) - raise LookupError("Not able to find release '{}' for {}".format(self.release, wf.full_name)) - - # Set outdir name if not defined - if not self.outdir: - self.outdir = "nf-core-{}".format(wf.name) - if self.release is not None: - self.outdir += "-{}".format(self.release) - - # Set the download URL and return - self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(wf.full_name, self.wf_sha) - return - - # If we got this far, must not be a nf-core pipeline - if self.pipeline.count("/") == 1: - # Looks like a GitHub address - try working with this repo - log.debug("Pipeline name looks like a GitHub address - attempting to download") - self.wf_name = self.pipeline - if not self.release: - self.release = "master" - self.wf_sha = self.release - if not self.outdir: - self.outdir = self.pipeline.replace("/", "-").lower() - if self.release is not None: - self.outdir += "-{}".format(self.release) - # Set the download URL and return - self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.release) + # Must not be a nf-core pipeline else: - log.error("Not able to find pipeline '{}'".format(self.pipeline)) - log.info( - "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) - ) - raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + if self.pipeline.count("/") == 1: + + # Looks like a GitHub address - try working with this repo + self.wf_name = self.pipeline + log.info( + f"Pipeline '{self.wf_name}' not in nf-core, but looks like a GitHub address - attempting anyway" + ) + + # Get releases from GitHub API + releases_url = f"https://api.github.com/repos/{self.wf_name}/releases" + releases_response = requests.get(releases_url) + self.wf_releases = list( + sorted(releases_response.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True) + ) + + else: + log.error("Not able to find pipeline '{}'".format(self.pipeline)) + log.info( + "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) + ) + raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + + # Get branch information from github api + branches_url = f"https://api.github.com/repos/{self.wf_name}/branches" + branch_response = requests.get(branches_url) + for branch in branch_response.json(): + if ( + branch["name"] != "TEMPLATE" + and branch["name"] != "initial_commit" + and not branch["name"].startswith("nf-core-template-merge") + ): + self.wf_branches[branch["name"]] = branch["commit"]["sha"] def download_wf_files(self): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" diff --git a/tests/test_download.py b/tests/test_download.py index eb14b3cf7..12fcfdfdf 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -195,7 +195,11 @@ def test_download_workflow_with_success(self, mock_download_image): tmp_dir = tempfile.mkdtemp() download_obj = DownloadWorkflow( - pipeline="nf-core/methylseq", outdir=os.path.join(tmp_dir, "new"), singularity=True + pipeline="nf-core/methylseq", + outdir=os.path.join(tmp_dir, "new"), + container="singularity", + release="dev", + compress="none", ) download_obj.download_workflow() From 60c3730875e855792c33516cfbd1bd2fb39950e6 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 08:19:11 +0200 Subject: [PATCH 25/45] Pipeline name should be full_name --- nf_core/download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 83465f0c8..3dbcfbac2 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -393,7 +393,7 @@ def fetch_workflow_details(self): if wf.full_name == self.pipeline or wf.name == self.pipeline: # Set pipeline name - self.wf_name = wf.name + self.wf_name = wf.full_name # Store releases self.wf_releases = list( From 9c17495c170ccdac093f36ec684e834f0c57cb56 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 13:07:20 +0200 Subject: [PATCH 26/45] Update nf_core/launch.py --- nf_core/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index 8f131939d..a25baf80e 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -187,7 +187,7 @@ def get_pipeline_schema(self): "Please select a release:", choices=release_tags, style=nf_core.utils.nfcore_question_style, - ).ask() + ).unsafe_ask() except LookupError: pass From 3d0d4157dbda335bab40c0690a9b0873c570adf1 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 23:14:54 +0200 Subject: [PATCH 27/45] Just bold, not white on bold. See nf-core/tools#1045 --- nf_core/download.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 3dbcfbac2..1f84975eb 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -289,15 +289,13 @@ def prompt_use_singularity_cachedir(self): "This allows downloaded images to be cached in a central location." ) if rich.prompt.Confirm.ask( - f"[blue bold]?[/] [white bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" + f"[blue bold]?[/] [bold]Define [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] for a shared Singularity image download folder?[/]" ): # Prompt user for a cache directory path cachedir_path = None while cachedir_path is None: cachedir_path = os.path.abspath( - rich.prompt.Prompt.ask( - "[blue bold]?[/] [white bold]Specify the path:[/] (leave blank to cancel)" - ) + rich.prompt.Prompt.ask("[blue bold]?[/] [bold]Specify the path:[/] (leave blank to cancel)") ) if cachedir_path == os.path.abspath(""): log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") @@ -321,7 +319,7 @@ def prompt_use_singularity_cachedir(self): f'[blue]export NXF_SINGULARITY_CACHEDIR="{cachedir_path}"[/]' ) append_to_file = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [white bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" + f"[blue bold]?[/] [bold]Add to [blue not bold]~/{os.path.basename(bashrc_path)}[/] ?[/]" ) if append_to_file: with open(os.path.expanduser(bashrc_path), "a") as f: @@ -349,7 +347,7 @@ def prompt_singularity_cachedir_only(self): "However if you will transfer the downloaded files to a different system then they should be copied to the target folder." ) self.singularity_cache_only = rich.prompt.Confirm.ask( - f"[blue bold]?[/] [white bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" + f"[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" ) # Sanity check From 10ddb0189fd805591e99eb005a6e8e85edb5f507 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 23:15:56 +0200 Subject: [PATCH 28/45] Removed more white bold, replaced with bold. Better on white colour scheme terminals. Fixes nf-core/tools#1045 --- nf_core/schema.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/schema.py b/nf_core/schema.py index 5196bcd8f..83f66a807 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -28,7 +28,7 @@ class PipelineSchema(object): functions to handle pipeline JSON Schema""" def __init__(self): - """ Initialise the object """ + """Initialise the object""" self.schema = None self.pipeline_dir = None @@ -46,7 +46,7 @@ def __init__(self): self.web_schema_build_api_url = None def get_schema_path(self, path, local_only=False, revision=None): - """ Given a pipeline name, directory, or path, set self.schema_filename """ + """Given a pipeline name, directory, or path, set self.schema_filename""" # Supplied path exists - assume a local pipeline directory or schema if os.path.exists(path): @@ -75,7 +75,7 @@ def get_schema_path(self, path, local_only=False, revision=None): raise AssertionError(error) def load_lint_schema(self): - """ Load and lint a given schema to see if it looks valid """ + """Load and lint a given schema to see if it looks valid""" try: self.load_schema() num_params = self.validate_schema() @@ -92,7 +92,7 @@ def load_lint_schema(self): raise AssertionError(error_msg) def load_schema(self): - """ Load a pipeline schema from a file """ + """Load a pipeline schema from a file""" with open(self.schema_filename, "r") as fh: self.schema = json.load(fh) self.schema_defaults = {} @@ -153,7 +153,7 @@ def get_schema_defaults(self): self.schema_defaults[p_key] = param["default"] def save_schema(self): - """ Save a pipeline schema to a file """ + """Save a pipeline schema to a file""" # Write results to a JSON file num_params = len(self.schema.get("properties", {})) num_params += sum([len(d.get("properties", {})) for d in self.schema.get("definitions", {}).values()]) @@ -189,7 +189,7 @@ def load_input_params(self, params_path): raise AssertionError(error_msg) def validate_params(self): - """ Check given parameters against a schema and validate """ + """Check given parameters against a schema and validate""" try: assert self.schema is not None jsonschema.validate(self.input_params, self.schema) @@ -317,7 +317,7 @@ def validate_schema_title_description(self, schema=None): ) def make_skeleton_schema(self): - """ Make a new pipeline schema from the template """ + """Make a new pipeline schema from the template""" self.schema_from_scratch = True # Use Jinja to render the template schema file to a variable env = jinja2.Environment( @@ -332,7 +332,7 @@ def make_skeleton_schema(self): self.get_schema_defaults() def build_schema(self, pipeline_dir, no_prompts, web_only, url): - """ Interactively build a new pipeline schema for a pipeline """ + """Interactively build a new pipeline schema for a pipeline""" if no_prompts: self.no_prompts = True @@ -476,7 +476,7 @@ def prompt_remove_schema_notfound_config(self, p_key): if self.no_prompts or self.schema_from_scratch: return True if Confirm.ask( - ":question: Unrecognised [white bold]'params.{}'[/] found in the schema but not in the pipeline config! [yellow]Remove it?".format( + ":question: Unrecognised [bold]'params.{}'[/] found in the schema but not in the pipeline config! [yellow]Remove it?".format( p_key ) ): @@ -497,7 +497,7 @@ def add_schema_found_configs(self): self.no_prompts or self.schema_from_scratch or Confirm.ask( - ":sparkles: Found [white bold]'params.{}'[/] in the pipeline config, but not in the schema. [blue]Add to pipeline schema?".format( + ":sparkles: Found [bold]'params.{}'[/] in the pipeline config, but not in the schema. [blue]Add to pipeline schema?".format( p_key ) ) From 06265eb6b943cb8dfc084daeda7b3166abf7fe2a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 23:21:18 +0200 Subject: [PATCH 29/45] Awesome questionary path auto-completion for singularity cachedir path --- nf_core/download.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nf_core/download.py b/nf_core/download.py index 1f84975eb..c67715170 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -295,7 +295,9 @@ def prompt_use_singularity_cachedir(self): cachedir_path = None while cachedir_path is None: cachedir_path = os.path.abspath( - rich.prompt.Prompt.ask("[blue bold]?[/] [bold]Specify the path:[/] (leave blank to cancel)") + questionary.path( + "Specify the path:", only_directories=True, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() ) if cachedir_path == os.path.abspath(""): log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") From c63487cb8083938fab1b729d1ab42a3be6d1430a Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 23:54:52 +0200 Subject: [PATCH 30/45] Refine cli flags, -c now for container instead of compress --- nf_core/__main__.py | 19 ++++++------------- nf_core/download.py | 4 ++-- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 9a56bad95..8630336ea 100755 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -208,25 +208,18 @@ def launch(pipeline, id, revision, command_only, params_in, params_out, save_all @click.option("-r", "--release", type=str, help="Pipeline release") @click.option("-o", "--outdir", type=str, help="Output directory") @click.option( - "-c", - "--compress", - type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), - help="Archive compression type", + "-x", "--compress", type=click.Choice(["tar.gz", "tar.bz2", "zip", "none"]), help="Archive compression type" ) @click.option("-f", "--force", is_flag=True, default=False, help="Overwrite existing files") @click.option( - "-i", - "--container", - type=click.Choice(["none", "singularity"]), - help="Download software container images", + "-c", "--container", type=click.Choice(["none", "singularity"]), help="Download software container images" ) @click.option( - "-s/-x", - "--singularity-cache/--no-singularity-cache", - help="Do / don't copy images to the output directory and set 'singularity.cacheDir' in workflow", + "--singularity-cache-only/--singularity-cache-copy", + help="Don't / do copy images to the output directory and set 'singularity.cacheDir' in workflow", ) @click.option("-p", "--parallel-downloads", type=int, default=4, help="Number of parallel image downloads") -def download(pipeline, release, outdir, compress, force, container, singularity_cache, parallel_downloads): +def download(pipeline, release, outdir, compress, force, container, singularity_cache_only, parallel_downloads): """ Download a pipeline, nf-core/configs and pipeline singularity images. @@ -234,7 +227,7 @@ def download(pipeline, release, outdir, compress, force, container, singularity_ workflow to use relative paths to the configs and singularity images. """ dl = nf_core.download.DownloadWorkflow( - pipeline, release, outdir, compress, force, container, singularity_cache, parallel_downloads + pipeline, release, outdir, compress, force, container, singularity_cache_only, parallel_downloads ) dl.download_workflow() diff --git a/nf_core/download.py b/nf_core/download.py index c67715170..6e6d14ab0 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -352,9 +352,9 @@ def prompt_singularity_cachedir_only(self): f"[blue bold]?[/] [bold]Copy singularity images from [blue not bold]$NXF_SINGULARITY_CACHEDIR[/] to the target folder?[/]" ) - # Sanity check + # Sanity check, for when passed as a cli flag if self.singularity_cache_only and self.container != "singularity": - raise LookupError("Command has '--singularity-cache' set, but '--container' is not 'singularity'") + raise LookupError("Command has '--singularity-cache-only' set, but '--container' is not 'singularity'") def prompt_compression_type(self): """Ask user if we should compress the downloaded files""" From 0f2b8f4e1fa5d90d06a5204648aa24581c092400 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 30 Apr 2021 23:59:30 +0200 Subject: [PATCH 31/45] Launch - sort releases by release date --- nf_core/launch.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/launch.py b/nf_core/launch.py index a25baf80e..216037477 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -22,7 +22,7 @@ class Launch(object): - """ Class to hold config option to launch a pipeline """ + """Class to hold config option to launch a pipeline""" def __init__( self, @@ -164,7 +164,7 @@ def launch_pipeline(self): self.launch_workflow() def get_pipeline_schema(self): - """ Load and validate the schema from the supplied pipeline """ + """Load and validate the schema from the supplied pipeline""" # Set up the schema self.schema_obj = nf_core.schema.PipelineSchema() @@ -246,7 +246,7 @@ def try_fetch_release_tags(self): if len(release_tags) == 0: log.error(f"Unable to find any release tags for {self.pipeline}. Will try to continue launch.") raise LookupError - release_tags = sorted(release_tags, key=lambda tag: tuple(tag.split(".")), reverse=True) + release_tags = sorted(release_tags, key=lambda tag: tag.get("published_at_timestamp", 0), reverse=True) return release_tags def set_schema_inputs(self): @@ -265,7 +265,7 @@ def set_schema_inputs(self): self.schema_obj.validate_params() def merge_nxf_flag_schema(self): - """ Take the Nextflow flag schema and merge it with the pipeline schema """ + """Take the Nextflow flag schema and merge it with the pipeline schema""" # Add the coreNextflow subschema to the schema definitions if "definitions" not in self.schema_obj.schema: self.schema_obj.schema["definitions"] = {} @@ -277,7 +277,7 @@ def merge_nxf_flag_schema(self): self.schema_obj.schema["allOf"].insert(0, {"$ref": "#/definitions/coreNextflow"}) def prompt_web_gui(self): - """ Ask whether to use the web-based or cli wizard to collect params """ + """Ask whether to use the web-based or cli wizard to collect params""" log.info( "[magenta]Would you like to enter pipeline parameters using a web-based interface or a command-line wizard?" ) @@ -292,7 +292,7 @@ def prompt_web_gui(self): return answer["use_web_gui"] == "Web based" def launch_web_gui(self): - """ Send schema to nf-core website and launch input GUI """ + """Send schema to nf-core website and launch input GUI""" content = { "post_content": "json_schema_launcher", @@ -397,7 +397,7 @@ def sanitise_web_response(self): params[param_id] = filter_func(params[param_id]) def prompt_schema(self): - """ Go through the pipeline schema and prompt user to change defaults """ + """Go through the pipeline schema and prompt user to change defaults""" answers = {} # Start with the subschema in the definitions - use order of allOf for allOf in self.schema_obj.schema.get("allOf", []): @@ -659,7 +659,7 @@ def print_param_header(self, param_id, param_obj, is_group=False): console.print("(Use arrow keys)", style="italic", highlight=False) def strip_default_params(self): - """ Strip parameters if they have not changed from the default """ + """Strip parameters if they have not changed from the default""" # Go through each supplied parameter (force list so we can delete in the loop) for param_id in list(self.schema_obj.input_params.keys()): @@ -683,7 +683,7 @@ def strip_default_params(self): del self.nxf_flags[param_id] def build_command(self): - """ Build the nextflow run command based on what we know """ + """Build the nextflow run command based on what we know""" # Core nextflow options for flag, val in self.nxf_flags.items(): @@ -717,7 +717,7 @@ def build_command(self): self.nextflow_cmd += ' --{} "{}"'.format(param, str(val).replace('"', '\\"')) def launch_workflow(self): - """ Launch nextflow if required """ + """Launch nextflow if required""" log.info("[bold underline]Nextflow command:[/]\n[magenta]{}\n\n".format(self.nextflow_cmd)) if Confirm.ask("Do you want to run this command now? "): From f1999d467ae3bdc3c16e04b4007c15fdf8b71cfc Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 00:15:09 +0200 Subject: [PATCH 32/45] Check if terminal is interactive before prompting for setting --- nf_core/download.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 6e6d14ab0..2664f0cf4 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -98,6 +98,8 @@ def __init__( self.parallel_downloads = parallel_downloads self.wf_name = None + self.wf_releases = {} + self.wf_branches = {} self.wf_sha = None self.wf_download_url = None self.nf_config = dict() @@ -106,8 +108,6 @@ def __init__( # Fetch remote workflows self.wfs = nf_core.list.Workflows() self.wfs.get_remote_workflows() - self.wf_releases = {} - self.wf_branches = {} def download_workflow(self): """Starts a nf-core workflow download.""" @@ -283,7 +283,11 @@ def prompt_container_download(self): def prompt_use_singularity_cachedir(self): """Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set""" - if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None: + if ( + self.container == "singularity" + and os.environ.get("NXF_SINGULARITY_CACHEDIR") is None + and stderr.is_interactive # Use rich auto-detection of interactive shells + ): stderr.print( "\nNextflow and nf-core can use an environment variable called [blue]$NXF_SINGULARITY_CACHEDIR[/] that is a path to a directory where remote Singularity images are stored. " "This allows downloaded images to be cached in a central location." From 683242d809a2dee105931974ef68d1c91656fef2 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 00:23:21 +0200 Subject: [PATCH 33/45] Screen container names for dynamic {squiggly_brackets} --- nf_core/download.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 2664f0cf4..621f8dabd 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -515,6 +515,7 @@ def find_container_images(self): """ log.debug("Fetching container names for workflow") + containers_raw = [] # Use linting code to parse the pipeline nextflow config self.nf_config = nf_core.utils.fetch_wf_config(os.path.join(self.outdir, "workflow")) @@ -522,7 +523,7 @@ def find_container_images(self): # Find any config variables that look like a container for k, v in self.nf_config.items(): if k.startswith("process.") and k.endswith(".container"): - self.containers.append(v.strip('"').strip("'")) + containers_raw.append(v.strip('"').strip("'")) # Recursive search through any DSL2 module files for container spec lines. for subdir, dirs, files in os.walk(os.path.join(self.outdir, "workflow", "modules")): @@ -539,15 +540,24 @@ def find_container_images(self): # If we have matches, save the first one that starts with http for m in matches: if m.startswith("http"): - self.containers.append(m.strip('"').strip("'")) + containers_raw.append(m.strip('"').strip("'")) break # If we get here then we didn't call break - just save the first match else: if len(matches) > 0: - self.containers.append(matches[0].strip('"').strip("'")) + containers_raw.append(matches[0].strip('"').strip("'")) # Remove duplicates and sort - self.containers = sorted(list(set(self.containers))) + containers_raw = sorted(list(set(containers_raw))) + + # Strip any container names that have dynamic names - eg. {params.foo} + self.containers = [] + for container in containers_raw: + if "{" in container and "}" in container: + log.error(f"Container name '{container}' has dynamic Nextflow logic in name - skipping") + log.info("Please use a 'nextflow run' command to fetch this container. Ask on Slack if you need help.") + else: + self.containers.append(container) log.info("Found {} container{}".format(len(self.containers), "s" if len(self.containers) > 1 else "")) From 7b745b3652d34b2c5e7538ff28fab178bc0b3a23 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 00:30:57 +0200 Subject: [PATCH 34/45] Push up minimum version of rich, louder colours --- nf_core/download.py | 4 +++- requirements.txt | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 621f8dabd..b62acfa98 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -554,7 +554,9 @@ def find_container_images(self): self.containers = [] for container in containers_raw: if "{" in container and "}" in container: - log.error(f"Container name '{container}' has dynamic Nextflow logic in name - skipping") + log.error( + f"[red]Container name [green]'{container}'[/] has dynamic Nextflow logic in name - skipping![/]" + ) log.info("Please use a 'nextflow run' command to fetch this container. Ask on Slack if you need help.") else: self.containers.append(container) diff --git a/requirements.txt b/requirements.txt index a8afe909c..3905fe6de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ pytest-workflow questionary>=1.8.0 requests_cache requests -rich>=9.8.2 -tabulate \ No newline at end of file +rich>=9.11.0 +tabulate From b66961c1528411f04804454e1304127621cc9f7b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 00:34:28 +0200 Subject: [PATCH 35/45] Bump rich minimum version again, as v1.10.0 fixed table style issue --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3905fe6de..33da40c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,5 +9,5 @@ pytest-workflow questionary>=1.8.0 requests_cache requests -rich>=9.11.0 +rich>=10.0.0 tabulate From a0bc438dfdf8b0b895e252df46094366fb0d7508 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 01:07:06 +0200 Subject: [PATCH 36/45] Fix pytests for download code --- nf_core/download.py | 18 +++++--- tests/test_download.py | 94 ++++++++++++++---------------------------- 2 files changed, 44 insertions(+), 68 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index b62acfa98..6e2532594 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -201,11 +201,11 @@ def prompt_pipeline_name(self): if self.pipeline.count("/") == 1: gh_response = requests.get(f"https://api.github.com/repos/{self.pipeline}") try: - assert gh_response.json().get("message") == "Not Found" + assert gh_response.json().get("message") != "Not Found" except AssertionError: - pass - else: raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + except AttributeError: + pass # When things are working we get a list, which doesn't work with .get() else: log.info( "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) @@ -419,16 +419,24 @@ def fetch_workflow_details(self): # Get releases from GitHub API releases_url = f"https://api.github.com/repos/{self.wf_name}/releases" releases_response = requests.get(releases_url) + + # Check that this repo existed + try: + assert releases_response.json().get("message") != "Not Found" + except AssertionError: + raise LookupError(f"Not able to find pipeline '{self.pipeline}'") + except AttributeError: + pass # When things are working we get a list, which doesn't work with .get() self.wf_releases = list( sorted(releases_response.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True) ) else: - log.error("Not able to find pipeline '{}'".format(self.pipeline)) + log.error(f"Not able to find pipeline '{self.pipeline}'") log.info( "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) ) - raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + raise LookupError(f"Not able to find pipeline '{self.pipeline}'") # Get branch information from github api branches_url = f"https://api.github.com/repos/{self.wf_name}/branches" diff --git a/tests/test_download.py b/tests/test_download.py index 12fcfdfdf..0ac72fede 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -20,70 +20,37 @@ class DownloadTest(unittest.TestCase): # # Tests for 'fetch_workflow_details()' # - @mock.patch("nf_core.list.RemoteWorkflow") - @mock.patch("nf_core.list.Workflows") - def test_fetch_workflow_details_for_release(self, mock_workflows, mock_workflow): - download_obj = DownloadWorkflow(pipeline="dummy", release="1.0.0") - mock_workflow.name = "dummy" - mock_workflow.releases = [{"tag_name": "1.0.0", "tag_sha": "n3v3rl4nd"}] - mock_workflows.remote_workflows = [mock_workflow] - - download_obj.fetch_workflow_details(mock_workflows) - - @mock.patch("nf_core.list.RemoteWorkflow") - @mock.patch("nf_core.list.Workflows") - def test_fetch_workflow_details_for_dev_version(self, mock_workflows, mock_workflow): - download_obj = DownloadWorkflow(pipeline="dummy") - mock_workflow.name = "dummy" - mock_workflow.releases = [] - mock_workflows.remote_workflows = [mock_workflow] - - download_obj.fetch_workflow_details(mock_workflows) - - @mock.patch("nf_core.list.RemoteWorkflow") - @mock.patch("nf_core.list.Workflows") - def test_fetch_workflow_details_and_autoset_release(self, mock_workflows, mock_workflow): - download_obj = DownloadWorkflow(pipeline="dummy") - mock_workflow.name = "dummy" - mock_workflow.releases = [{"tag_name": "1.0.0", "tag_sha": "n3v3rl4nd"}] - mock_workflows.remote_workflows = [mock_workflow] - - download_obj.fetch_workflow_details(mock_workflows) - assert download_obj.release == "1.0.0" + def test_fetch_workflow_details_for_nf_core(self): + download_obj = DownloadWorkflow(pipeline="methylseq") + download_obj.fetch_workflow_details() + assert download_obj.wf_name == "nf-core/methylseq" + for r in download_obj.wf_releases: + if r.get("tag_name") == "1.6": + break + else: + raise AssertionError("Release 1.6 not found") + assert "dev" in download_obj.wf_branches.keys() + + def test_fetch_workflow_details_for_not_nf_core(self): + download_obj = DownloadWorkflow(pipeline="ewels/MultiQC") + download_obj.fetch_workflow_details() + assert download_obj.wf_name == "ewels/MultiQC" + for r in download_obj.wf_releases: + if r.get("tag_name") == "v1.10": + break + else: + raise AssertionError("MultiQC release v1.10 not found") + assert "master" in download_obj.wf_branches.keys() - @mock.patch("nf_core.list.RemoteWorkflow") - @mock.patch("nf_core.list.Workflows") @pytest.mark.xfail(raises=LookupError, strict=True) - def test_fetch_workflow_details_for_unknown_release(self, mock_workflows, mock_workflow): - download_obj = DownloadWorkflow(pipeline="dummy", release="1.2.0") - mock_workflow.name = "dummy" - mock_workflow.releases = [{"tag_name": "1.0.0", "tag_sha": "n3v3rl4nd"}] - mock_workflows.remote_workflows = [mock_workflow] - - download_obj.fetch_workflow_details(mock_workflows) + def test_fetch_workflow_details_not_exists(self): + download_obj = DownloadWorkflow(pipeline="made_up_pipeline") + download_obj.fetch_workflow_details() - @mock.patch("nf_core.list.Workflows") - def test_fetch_workflow_details_for_github_ressource(self, mock_workflows): - download_obj = DownloadWorkflow(pipeline="myorg/dummy", release="1.2.0") - mock_workflows.remote_workflows = [] - - download_obj.fetch_workflow_details(mock_workflows) - - @mock.patch("nf_core.list.Workflows") - def test_fetch_workflow_details_for_github_ressource_take_master(self, mock_workflows): - download_obj = DownloadWorkflow(pipeline="myorg/dummy") - mock_workflows.remote_workflows = [] - - download_obj.fetch_workflow_details(mock_workflows) - assert download_obj.release == "master" - - @mock.patch("nf_core.list.Workflows") @pytest.mark.xfail(raises=LookupError, strict=True) - def test_fetch_workflow_details_no_search_result(self, mock_workflows): - download_obj = DownloadWorkflow(pipeline="http://my-server.org/dummy", release="1.2.0") - mock_workflows.remote_workflows = [] - - download_obj.fetch_workflow_details(mock_workflows) + def test_fetch_workflow_details_not_exists_slash(self): + download_obj = DownloadWorkflow(pipeline="made-up/pipeline") + download_obj.fetch_workflow_details() # # Tests for 'download_wf_files' @@ -190,7 +157,8 @@ def test_singularity_pull_image(self, mock_rich_progress): # Tests for the main entry method 'download_workflow' # @mock.patch("nf_core.download.DownloadWorkflow.singularity_pull_image") - def test_download_workflow_with_success(self, mock_download_image): + @mock.patch("shutil.which") + def test_download_workflow_with_success(self, mock_download_image, mock_singularity_installed): tmp_dir = tempfile.mkdtemp() @@ -198,8 +166,8 @@ def test_download_workflow_with_success(self, mock_download_image): pipeline="nf-core/methylseq", outdir=os.path.join(tmp_dir, "new"), container="singularity", - release="dev", - compress="none", + release="1.6", + compress_type="none", ) download_obj.download_workflow() From bd8152becd9a982d5aab2098af93059b0a368e82 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 01:17:18 +0200 Subject: [PATCH 37/45] Don't force interactive terminal on GitHub Actions for pytest --- tests/test_download.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_download.py b/tests/test_download.py index 0ac72fede..db90db60f 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -161,6 +161,10 @@ def test_singularity_pull_image(self, mock_rich_progress): def test_download_workflow_with_success(self, mock_download_image, mock_singularity_installed): tmp_dir = tempfile.mkdtemp() + try: + del os.environ["GITHUB_ACTIONS"] + except KeyError: + pass download_obj = DownloadWorkflow( pipeline="nf-core/methylseq", From 9bbdbe3fac7d9aa65a13a0186d60ded53cd6cb6f Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 01:22:13 +0200 Subject: [PATCH 38/45] Different approach to try to avoid interactive prompt in tests --- tests/test_download.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index db90db60f..0009a1974 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -161,10 +161,7 @@ def test_singularity_pull_image(self, mock_rich_progress): def test_download_workflow_with_success(self, mock_download_image, mock_singularity_installed): tmp_dir = tempfile.mkdtemp() - try: - del os.environ["GITHUB_ACTIONS"] - except KeyError: - pass + os.environ["NXF_SINGULARITY_CACHEDIR"] = "foo" download_obj = DownloadWorkflow( pipeline="nf-core/methylseq", From 67ec08bca1d045419f8d3b0156c36399f85018f7 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 22:38:22 +0200 Subject: [PATCH 39/45] Refactor: Move bunch of prompt / lookup code into utils for reuse --- nf_core/download.py | 142 +++++++------------------------------------ nf_core/utils.py | 145 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+), 121 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 6e2532594..970d3fca7 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -97,7 +97,6 @@ def __init__( self.singularity_cache_only = singularity_cache_only self.parallel_downloads = parallel_downloads - self.wf_name = None self.wf_releases = {} self.wf_branches = {} self.wf_sha = None @@ -115,14 +114,14 @@ def download_workflow(self): # Get workflow details try: self.prompt_pipeline_name() - self.fetch_workflow_details() + self.wf_releases, self.wf_branches = nf_core.utils.get_repo_releases_branches(self.pipeline, self.wfs) self.prompt_release() self.get_release_hash() self.prompt_container_download() self.prompt_use_singularity_cachedir() self.prompt_singularity_cachedir_only() self.prompt_compression_type() - except LookupError as e: + except AssertionError as e: log.critical(e) sys.exit(1) @@ -165,7 +164,12 @@ def download_workflow(self): # Download the centralised configs log.info("Downloading centralised configs from GitHub") self.download_configs() - self.wf_use_local_configs() + try: + self.wf_use_local_configs() + except FileNotFoundError as e: + log.error("Error editing pipeline config file to use local configs!") + log.critical(e) + sys.exit(1) # Download the singularity images if self.container == "singularity": @@ -186,54 +190,13 @@ def prompt_pipeline_name(self): if self.pipeline is None: stderr.print("Specify the name of a nf-core pipeline or a GitHub repository name (user/repo).") - self.pipeline = questionary.autocomplete( - "Pipeline name:", - choices=[wf.name for wf in self.wfs.remote_workflows], - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - - # Fast-fail for unrecognised pipelines (we check again at the end) - for wf in self.wfs.remote_workflows: - if wf.full_name == self.pipeline or wf.name == self.pipeline: - break - else: - # Non nf-core GitHub repo - if self.pipeline.count("/") == 1: - gh_response = requests.get(f"https://api.github.com/repos/{self.pipeline}") - try: - assert gh_response.json().get("message") != "Not Found" - except AssertionError: - raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) - except AttributeError: - pass # When things are working we get a list, which doesn't work with .get() - else: - log.info( - "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) - ) - raise LookupError("Not able to find pipeline '{}'".format(self.pipeline)) + self.pipeline = nf_core.utils.prompt_remote_pipeline_name(self.wfs) def prompt_release(self): """Prompt for pipeline release / branch""" # Prompt user for release tag if '--release' was not set if self.release is None: - choices = [] - - # Releases - if len(self.wf_releases) > 0: - for tag in map(lambda release: release.get("tag_name"), self.wf_releases): - tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] - choices.append(questionary.Choice(title=tag_display, value=tag)) - - # Branches - for branch in self.wf_branches.keys(): - branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] - choices.append(questionary.Choice(title=branch_display, value=branch)) - - if len(choices) > 0: - stderr.print("\nChoose the release or branch that should be downloaded.") - self.release = questionary.select( - "Select release / branch:", choices=choices, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() + self.release = nf_core.utils.prompt_pipeline_release_branch(self.wf_releases, self.wf_branches) def get_release_hash(self): """Find specified release / branch hash""" @@ -245,27 +208,28 @@ def get_release_hash(self): # Release else: for r in self.wf_releases: - if r["tag_name"] == self.release.lstrip("v"): + if r["tag_name"] == self.release: self.wf_sha = r["tag_sha"] break # Can't find the release or branch - throw an error else: - log.error("Not able to find release '{}' for {}".format(self.release, self.wf_name)) log.info( - "Available {} releases: {}".format( - self.wf_name, ", ".join([r["tag_name"] for r in self.wf_releases]) + "Available {} releases: '{}'".format( + self.pipeline, "', '".join([r["tag_name"] for r in self.wf_releases]) ) ) - log.info("Available {} branches: '{}'".format(self.wf_name, "', '".join(self.wf_branches.keys()))) - raise LookupError("Not able to find release / branch '{}' for {}".format(self.release, self.wf_name)) + log.info("Available {} branches: '{}'".format(self.pipeline, "', '".join(self.wf_branches.keys()))) + raise AssertionError( + "Not able to find release / branch '{}' for {}".format(self.release, self.pipeline) + ) # Set the outdir if not self.outdir: - self.outdir = "{}-{}".format(self.wf_name.replace("/", "-").lower(), self.release) + self.outdir = "{}-{}".format(self.pipeline.replace("/", "-").lower(), self.release) # Set the download URL and return - self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.wf_name, self.wf_sha) + self.wf_download_url = "https://github.com/{}/archive/{}.zip".format(self.pipeline, self.wf_sha) def prompt_container_download(self): """Prompt whether to download container images or not""" @@ -358,7 +322,7 @@ def prompt_singularity_cachedir_only(self): # Sanity check, for when passed as a cli flag if self.singularity_cache_only and self.container != "singularity": - raise LookupError("Command has '--singularity-cache-only' set, but '--container' is not 'singularity'") + raise AssertionError("Command has '--singularity-cache-only' set, but '--container' is not 'singularity'") def prompt_compression_type(self): """Ask user if we should compress the downloaded files""" @@ -385,70 +349,6 @@ def prompt_compression_type(self): if self.compress_type == "none": self.compress_type = None - def fetch_workflow_details(self): - """Fetches details of a nf-core workflow to download. - - Raises: - LockupError, if the pipeline can not be found. - """ - - # Get workflow download details - for wf in self.wfs.remote_workflows: - if wf.full_name == self.pipeline or wf.name == self.pipeline: - - # Set pipeline name - self.wf_name = wf.full_name - - # Store releases - self.wf_releases = list( - sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - ) - - break - - # Must not be a nf-core pipeline - else: - if self.pipeline.count("/") == 1: - - # Looks like a GitHub address - try working with this repo - self.wf_name = self.pipeline - log.info( - f"Pipeline '{self.wf_name}' not in nf-core, but looks like a GitHub address - attempting anyway" - ) - - # Get releases from GitHub API - releases_url = f"https://api.github.com/repos/{self.wf_name}/releases" - releases_response = requests.get(releases_url) - - # Check that this repo existed - try: - assert releases_response.json().get("message") != "Not Found" - except AssertionError: - raise LookupError(f"Not able to find pipeline '{self.pipeline}'") - except AttributeError: - pass # When things are working we get a list, which doesn't work with .get() - self.wf_releases = list( - sorted(releases_response.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True) - ) - - else: - log.error(f"Not able to find pipeline '{self.pipeline}'") - log.info( - "Available nf-core pipelines: '{}'".format("', '".join([w.name for w in self.wfs.remote_workflows])) - ) - raise LookupError(f"Not able to find pipeline '{self.pipeline}'") - - # Get branch information from github api - branches_url = f"https://api.github.com/repos/{self.wf_name}/branches" - branch_response = requests.get(branches_url) - for branch in branch_response.json(): - if ( - branch["name"] != "TEMPLATE" - and branch["name"] != "initial_commit" - and not branch["name"].startswith("nf-core-template-merge") - ): - self.wf_branches[branch["name"]] = branch["commit"]["sha"] - def download_wf_files(self): """Downloads workflow files from GitHub to the :attr:`self.outdir`.""" log.debug("Downloading {}".format(self.wf_download_url)) @@ -459,7 +359,7 @@ def download_wf_files(self): zipfile.extractall(self.outdir) # Rename the internal directory name to be more friendly - gh_name = "{}-{}".format(self.wf_name, self.wf_sha).split("/")[-1] + gh_name = "{}-{}".format(self.pipeline, self.wf_sha).split("/")[-1] os.rename(os.path.join(self.outdir, gh_name), os.path.join(self.outdir, "workflow")) # Make downloaded files executable diff --git a/nf_core/utils.py b/nf_core/utils.py index 2670f0310..0cd3e24d7 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -13,6 +13,7 @@ import logging import os import prompt_toolkit +import questionary import re import requests import requests_cache @@ -550,3 +551,147 @@ def write_line_break(self, data=None): CustomDumper.add_representer(dict, CustomDumper.represent_dict_preserve_order) return CustomDumper + + +def prompt_remote_pipeline_name(wfs): + """Prompt for the pipeline name with questionary + + Args: + wfs: A nf_core.list.Workflows() object, where get_remote_workflows() has been called. + + Returns: + pipeline (str): GitHub repo - username/repo + + Raises: + AssertionError, if pipeline cannot be found + """ + + pipeline = questionary.autocomplete( + "Pipeline name:", + choices=[wf.name for wf in wfs.remote_workflows], + style=nfcore_question_style, + ).unsafe_ask() + + # Check nf-core repos + for wf in wfs.remote_workflows: + if wf.full_name == pipeline or wf.name == pipeline: + return wf.full_name + + # Non nf-core repo on GitHub + else: + if pipeline.count("/") == 1: + try: + gh_response = requests.get(f"https://api.github.com/repos/{pipeline}") + assert gh_response.json().get("message") != "Not Found" + except AssertionError: + pass + else: + return pipeline + + log.info("Available nf-core pipelines: '{}'".format("', '".join([w.name for w in wfs.remote_workflows]))) + raise AssertionError(f"Not able to find pipeline '{pipeline}'") + + +def prompt_pipeline_release_branch(wf_releases, wf_branches): + """Prompt for pipeline release / branch + + Args: + wf_releases (array): Array of repo releases as returned by the GitHub API + wf_branches (array): Array of repo branches, as returned by the GitHub API + + Returns: + choice (str): Selected release / branch name + """ + # Prompt user for release tag + choices = [] + + # Releases + if len(wf_releases) > 0: + for tag in map(lambda release: release.get("tag_name"), wf_releases): + tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] + choices.append(questionary.Choice(title=tag_display, value=tag)) + + # Branches + for branch in wf_branches.keys(): + branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] + choices.append(questionary.Choice(title=branch_display, value=branch)) + + if len(choices) == 0: + return False + + return questionary.select("Select release / branch:", choices=choices, style=nfcore_question_style).unsafe_ask() + + +def get_repo_releases_branches(pipeline, wfs): + """Fetches details of a nf-core workflow to download. + + Args: + pipeline (str): GitHub repo username/repo + wfs: A nf_core.list.Workflows() object, where get_remote_workflows() has been called. + + Returns: + wf_releases, wf_branches (tuple): Array of releases, Array of branches + + Raises: + LockupError, if the pipeline can not be found. + """ + + wf_releases = [] + wf_branches = {} + + # Repo is a nf-core pipeline + for wf in wfs.remote_workflows: + if wf.full_name == pipeline or wf.name == pipeline: + + # Set to full name just in case it didn't have the nf-core/ prefix + pipeline = wf.full_name + + # Store releases and stop loop + wf_releases = list(sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) + break + + # Arbitrary GitHub repo + else: + if pipeline.count("/") == 1: + + # Looks like a GitHub address - try working with this repo + log.debug( + f"Pipeline '{pipeline}' not in nf-core, but looks like a GitHub address - fetching releases from API" + ) + + # Get releases from GitHub API + rel_r = requests.get(f"https://api.github.com/repos/{pipeline}/releases") + + # Check that this repo existed + try: + assert rel_r.json().get("message") != "Not Found" + except AssertionError: + raise AssertionError(f"Not able to find pipeline '{pipeline}'") + except AttributeError: + # When things are working we get a list, which doesn't work with .get() + wf_releases = list(sorted(rel_r.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) + + # Get release tag commit hashes + if len(wf_releases) > 0: + # Get commit hash information for each release + tags_r = requests.get(f"https://api.github.com/repos/{pipeline}/tags") + for tag in tags_r.json(): + for release in wf_releases: + if tag["name"] == release["tag_name"]: + release["tag_sha"] = tag["commit"]["sha"] + + else: + log.info("Available nf-core pipelines: '{}'".format("', '".join([w.name for w in wfs.remote_workflows]))) + raise AssertionError(f"Not able to find pipeline '{pipeline}'") + + # Get branch information from github api - should be no need to check if the repo exists again + branch_response = requests.get(f"https://api.github.com/repos/{pipeline}/branches") + for branch in branch_response.json(): + if ( + branch["name"] != "TEMPLATE" + and branch["name"] != "initial_commit" + and not branch["name"].startswith("nf-core-template-merge") + ): + wf_branches[branch["name"]] = branch["commit"]["sha"] + + return wf_releases, wf_branches From ed9dd62f99432448d2a5b37993f2b55b8687d658 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Sat, 1 May 2021 23:03:18 +0200 Subject: [PATCH 40/45] Launch: Prompt for pipeline and release --- nf_core/download.py | 5 +-- nf_core/launch.py | 90 ++++++++++++++++++++------------------------- 2 files changed, 40 insertions(+), 55 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 970d3fca7..591fcad29 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -238,10 +238,7 @@ def prompt_container_download(self): stderr.print("\nIn addition to the pipeline code, this tool can download software containers.") self.container = questionary.select( "Download software container images:", - choices=[ - "none", - "singularity", - ], + choices=["none", "singularity"], style=nf_core.utils.nfcore_question_style, ).unsafe_ask() diff --git a/nf_core/launch.py b/nf_core/launch.py index 216037477..8547554ca 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -57,7 +57,11 @@ def __init__( if self.web_id: self.web_schema_launch_web_url = "{}?id={}".format(self.web_schema_launch_url, web_id) self.web_schema_launch_api_url = "{}?id={}&api=true".format(self.web_schema_launch_url, web_id) - self.nextflow_cmd = "nextflow run {}".format(self.pipeline) + self.nextflow_cmd = None + + # Fetch remote workflows + self.wfs = nf_core.list.Workflows() + self.wfs.get_remote_workflows() # Prepend property names with a single hyphen in case we have parameters with the same ID self.nxf_flag_schema = { @@ -94,12 +98,24 @@ def __init__( def launch_pipeline(self): - # Check that we have everything we need + # Prompt for pipeline if not supplied and no web launch ID if self.pipeline is None and self.web_id is None: - log.error( - "Either a pipeline name or web cache ID is required. Please see nf-core launch --help for more information." - ) - return False + launch_type = questionary.select( + "Launch local pipeline or remote GitHub pipeline?", + choices=["Remote pipeline", "Local path"], + style=nf_core.utils.nfcore_question_style, + ).unsafe_ask() + + if launch_type == "Remote pipeline": + try: + self.pipeline = nf_core.utils.prompt_remote_pipeline_name(self.wfs) + except AssertionError as e: + log.error(e.args[0]) + return False + else: + self.pipeline = questionary.path( + "Path to workflow:", style=nf_core.utils.nfcore_question_style + ).unsafe_ask() # Check if the output file exists already if os.path.exists(self.params_out): @@ -111,7 +127,9 @@ def launch_pipeline(self): log.info("Exiting. Use --params-out to specify a custom filename.") return False - log.info("This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n") + log.info( + "NOTE: This tool ignores any pipeline parameter defaults overwritten by Nextflow config files or profiles\n" + ) # Check if we have a web ID if self.web_id is not None: @@ -170,29 +188,25 @@ def get_pipeline_schema(self): self.schema_obj = nf_core.schema.PipelineSchema() # Check if this is a local directory - if os.path.exists(self.pipeline): + localpath = os.path.abspath(os.path.expanduser(self.pipeline)) + if os.path.exists(localpath): # Set the nextflow launch command to use full paths - self.nextflow_cmd = "nextflow run {}".format(os.path.abspath(self.pipeline)) + self.pipeline = localpath + self.nextflow_cmd = f"nextflow run {localpath}" else: # Assume nf-core if no org given if self.pipeline.count("/") == 0: - self.nextflow_cmd = "nextflow run nf-core/{}".format(self.pipeline) + self.pipeline = f"nf-core/{self.pipeline}" + self.nextflow_cmd = "nextflow run {}".format(self.pipeline) if not self.pipeline_revision: - check_for_releases = Confirm.ask("Would you like to select a specific release?") - if check_for_releases: - try: - release_tags = self.try_fetch_release_tags() - self.pipeline_revision = questionary.select( - "Please select a release:", - choices=release_tags, - style=nf_core.utils.nfcore_question_style, - ).unsafe_ask() - except LookupError: - pass - - # Add revision flag to commands if set - if self.pipeline_revision: + try: + wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches(self.pipeline, self.wfs) + except AssertionError as e: + log.error(e) + return False + + self.pipeline_revision = nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches) self.nextflow_cmd += " -r {}".format(self.pipeline_revision) # Get schema from name, load it and lint it @@ -208,7 +222,7 @@ def get_pipeline_schema(self): if not os.path.exists(os.path.join(self.schema_obj.pipeline_dir, "nextflow.config")) and not os.path.exists( os.path.join(self.schema_obj.pipeline_dir, "main.nf") ): - log.error("Could not find a main.nf or nextfow.config file, are you sure this is a pipeline?") + log.error("Could not find a 'main.nf' or 'nextflow.config' file, are you sure this is a pipeline?") return False # Build a schema for this pipeline @@ -223,32 +237,6 @@ def get_pipeline_schema(self): log.error("Could not build pipeline schema: {}".format(e)) return False - def try_fetch_release_tags(self): - """Tries to fetch tag names of pipeline releases from github - - Returns: - release_tags (list[str]): Returns list of release tags - - Raises: - LookupError, if no releases were found - """ - # Fetch releases from github api - releases_url = "https://api.github.com/repos/nf-core/{}/releases".format(self.pipeline) - response = requests.get(releases_url) - if not response.ok: - log.error(f"Unable to find any release tags for {self.pipeline}. Will try to continue launch.") - raise LookupError - - # Filter out the release tags and sort them - release_tags = map(lambda release: release.get("tag_name", None), response.json()) - release_tags = filter(lambda tag: tag != None, release_tags) - release_tags = list(release_tags) - if len(release_tags) == 0: - log.error(f"Unable to find any release tags for {self.pipeline}. Will try to continue launch.") - raise LookupError - release_tags = sorted(release_tags, key=lambda tag: tag.get("published_at_timestamp", 0), reverse=True) - return release_tags - def set_schema_inputs(self): """ Take the loaded schema and set the defaults as the input parameters From 8d4fc0e3400100e36c4a439035fa6f9b4a07c6d0 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 6 May 2021 22:58:29 +0200 Subject: [PATCH 41/45] nf_core.utils.get_repo_releases_branches() - return pipeline name. Sometimes we add the nf-core/ prefix to the pipeline name, so return that too. --- nf_core/download.py | 4 +++- nf_core/launch.py | 4 +++- nf_core/utils.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 591fcad29..78c666610 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -114,7 +114,9 @@ def download_workflow(self): # Get workflow details try: self.prompt_pipeline_name() - self.wf_releases, self.wf_branches = nf_core.utils.get_repo_releases_branches(self.pipeline, self.wfs) + self.pipeline, self.wf_releases, self.wf_branches = nf_core.utils.get_repo_releases_branches( + self.pipeline, self.wfs + ) self.prompt_release() self.get_release_hash() self.prompt_container_download() diff --git a/nf_core/launch.py b/nf_core/launch.py index 8547554ca..557001598 100644 --- a/nf_core/launch.py +++ b/nf_core/launch.py @@ -201,7 +201,9 @@ def get_pipeline_schema(self): if not self.pipeline_revision: try: - wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches(self.pipeline, self.wfs) + self.pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches( + self.pipeline, self.wfs + ) except AssertionError as e: log.error(e) return False diff --git a/nf_core/utils.py b/nf_core/utils.py index 7724cedb1..f40683cbd 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -711,4 +711,5 @@ def get_repo_releases_branches(pipeline, wfs): ): wf_branches[branch["name"]] = branch["commit"]["sha"] - return wf_releases, wf_branches + # Return pipeline again in case we added the nf-core/ prefix + return pipeline, wf_releases, wf_branches From 5478af857e1d3e1172a1401f40c084106feb3047 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 6 May 2021 23:04:38 +0200 Subject: [PATCH 42/45] Update pytests --- tests/test_download.py | 101 +++++++++++++++++++++++++---------------- tests/test_utils.py | 35 ++++++++++++++ 2 files changed, 98 insertions(+), 38 deletions(-) diff --git a/tests/test_download.py b/tests/test_download.py index 0009a1974..c12bafe75 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -18,56 +18,81 @@ class DownloadTest(unittest.TestCase): # - # Tests for 'fetch_workflow_details()' - # - def test_fetch_workflow_details_for_nf_core(self): - download_obj = DownloadWorkflow(pipeline="methylseq") - download_obj.fetch_workflow_details() - assert download_obj.wf_name == "nf-core/methylseq" - for r in download_obj.wf_releases: - if r.get("tag_name") == "1.6": - break - else: - raise AssertionError("Release 1.6 not found") - assert "dev" in download_obj.wf_branches.keys() - - def test_fetch_workflow_details_for_not_nf_core(self): - download_obj = DownloadWorkflow(pipeline="ewels/MultiQC") - download_obj.fetch_workflow_details() - assert download_obj.wf_name == "ewels/MultiQC" - for r in download_obj.wf_releases: - if r.get("tag_name") == "v1.10": - break - else: - raise AssertionError("MultiQC release v1.10 not found") - assert "master" in download_obj.wf_branches.keys() - - @pytest.mark.xfail(raises=LookupError, strict=True) - def test_fetch_workflow_details_not_exists(self): - download_obj = DownloadWorkflow(pipeline="made_up_pipeline") - download_obj.fetch_workflow_details() - - @pytest.mark.xfail(raises=LookupError, strict=True) - def test_fetch_workflow_details_not_exists_slash(self): - download_obj = DownloadWorkflow(pipeline="made-up/pipeline") - download_obj.fetch_workflow_details() + # Tests for 'get_release_hash' + # + def test_get_release_hash_release(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline = "methylseq" + download_obj = DownloadWorkflow(pipeline=pipeline, release="1.6") + ( + download_obj.pipeline, + download_obj.wf_releases, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_release_hash() + assert download_obj.wf_sha == "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + assert download_obj.outdir == "nf-core-methylseq-1.6" + assert ( + download_obj.wf_download_url + == "https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip" + ) + + def test_get_release_hash_branch(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + # Exoseq pipeline is archived, so `dev` branch should be stable + pipeline = "exoseq" + download_obj = DownloadWorkflow(pipeline=pipeline, release="dev") + ( + download_obj.pipeline, + download_obj.wf_releases, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_release_hash() + assert download_obj.wf_sha == "819cbac792b76cf66c840b567ed0ee9a2f620db7" + assert download_obj.outdir == "nf-core-exoseq-dev" + assert ( + download_obj.wf_download_url + == "https://github.com/nf-core/exoseq/archive/819cbac792b76cf66c840b567ed0ee9a2f620db7.zip" + ) + + @pytest.mark.xfail(raises=AssertionError, strict=True) + def test_get_release_hash_non_existent_release(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline = "methylseq" + download_obj = DownloadWorkflow(pipeline=pipeline, release="thisisfake") + ( + download_obj.pipeline, + download_obj.wf_releases, + download_obj.wf_branches, + ) = nf_core.utils.get_repo_releases_branches(pipeline, wfs) + download_obj.get_release_hash() # # Tests for 'download_wf_files' # def test_download_wf_files(self): - download_obj = DownloadWorkflow(pipeline="dummy", release="1.2.0", outdir=tempfile.mkdtemp()) - download_obj.wf_name = "nf-core/methylseq" - download_obj.wf_sha = "1.0" - download_obj.wf_download_url = "https://github.com/nf-core/methylseq/archive/1.0.zip" + outdir = tempfile.mkdtemp() + download_obj = DownloadWorkflow(pipeline="nf-core/methylseq", release="1.6") + download_obj.outdir = outdir + download_obj.wf_sha = "b3e5e3b95aaf01d98391a62a10a3990c0a4de395" + download_obj.wf_download_url = ( + "https://github.com/nf-core/methylseq/archive/b3e5e3b95aaf01d98391a62a10a3990c0a4de395.zip" + ) download_obj.download_wf_files() + assert os.path.exists(os.path.join(outdir, "workflow", "main.nf")) # # Tests for 'download_configs' # def test_download_configs(self): - download_obj = DownloadWorkflow(pipeline="dummy", release="1.2.0", outdir=tempfile.mkdtemp()) + outdir = tempfile.mkdtemp() + download_obj = DownloadWorkflow(pipeline="nf-core/methylseq", release="1.6") + download_obj.outdir = outdir download_obj.download_configs() + assert os.path.exists(os.path.join(outdir, "configs", "nfcore_custom.config")) # # Tests for 'wf_use_local_configs' diff --git a/tests/test_utils.py b/tests/test_utils.py index c6947861c..e016f14ab 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,7 @@ """ import nf_core.create +import nf_core.list import nf_core.utils import mock @@ -132,3 +133,37 @@ def test_pip_erroneous_package(self): """Tests the PyPi API package information query""" with pytest.raises(ValueError): nf_core.utils.pip_package("not_a_package=1.0") + + def test_get_repo_releases_branches_nf_core(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("methylseq", wfs) + for r in wf_releases: + if r.get("tag_name") == "1.6": + break + else: + raise AssertionError("Release 1.6 not found") + assert "dev" in wf_branches.keys() + + def test_get_repo_releases_branches_not_nf_core(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("ewels/MultiQC", wfs) + for r in wf_releases: + if r.get("tag_name") == "v1.10": + break + else: + raise AssertionError("MultiQC release v1.10 not found") + assert "master" in wf_branches.keys() + + @pytest.mark.xfail(raises=AssertionError, strict=True) + def test_get_repo_releases_branches_not_exists(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("made_up_pipeline", wfs) + + @pytest.mark.xfail(raises=AssertionError, strict=True) + def test_get_repo_releases_branches_not_exists_slash(self): + wfs = nf_core.list.Workflows() + wfs.get_remote_workflows() + pipeline, wf_releases, wf_branches = nf_core.utils.get_repo_releases_branches("made-up/pipeline", wfs) From 73fe7f420b6058be542193350157563ff623606b Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 6 May 2021 23:43:07 +0200 Subject: [PATCH 43/45] Download - update readme docs --- README.md | 63 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 67f55fb6e..670ffebd6 100644 --- a/README.md +++ b/README.md @@ -344,15 +344,14 @@ Sometimes you may need to run an nf-core pipeline on a server or HPC system that In this case you will need to fetch the pipeline files first, then manually transfer them to your system. To make this process easier and ensure accurate retrieval of correctly versioned code and software containers, we have written a download helper tool. -Simply specify the name of the nf-core pipeline and it will be downloaded to your current working directory. -By default, the pipeline will download the pipeline code and the [institutional nf-core/configs](https://github.com/nf-core/configs) files. -If you specify the flag `--singularity`, it will also download any singularity image files that are required. +The `nf-core download` command will download both the pipeline code and the [institutional nf-core/configs](https://github.com/nf-core/configs) files. It can also optionally download any singularity image files that are required. -Use `-r`/`--release` to download a specific release of the pipeline. If not specified, the tool will automatically fetch the latest release. +If run without any arguments, the download tool will interactively prompt you for the required information. +Each option has a flag, if all are supplied then it will run without any user input needed. ```console -$ nf-core download rnaseq -r 3.0 --singularity +$ nf-core download ,--./,-. ___ __ __ __ ___ /,-._.--~\ @@ -360,33 +359,46 @@ $ nf-core download rnaseq -r 3.0 --singularity | \| | \__, \__/ | \ |___ \`-._,-`-, `._,._,' - nf-core/tools version 1.13 + nf-core/tools version 1.14 + + +Specify the name of a nf-core pipeline or a GitHub repository name (user/repo). +? Pipeline name: rnaseq +? Select release / branch: 3.0 [release] +In addition to the pipeline code, this tool can download software containers. +? Download software container images: singularity +Nextflow and nf-core can use an environment variable called $NXF_SINGULARITY_CACHEDIR that is a path to a directory where remote Singularity +images are stored. This allows downloaded images to be cached in a central location. +? Define $NXF_SINGULARITY_CACHEDIR for a shared Singularity image download folder? [y/n]: y +? Specify the path: cachedir/ -INFO Saving rnaseq +So that $NXF_SINGULARITY_CACHEDIR is always defined, you can add it to your ~/.bashrc file. This will then be autmoatically set every time you open a new terminal. We can add the following line to this file for you: +export NXF_SINGULARITY_CACHEDIR="/path/to/demo/cachedir" +? Add to ~/.bashrc ? [y/n]: n + +If transferring the downloaded files to another system, it can be convenient to have everything compressed in a single file. +This is not recommended when downloading Singularity images, as it can take a long time and saves very little space. +? Choose compression type: none +INFO Saving 'nf-core/rnaseq Pipeline release: '3.0' - Pull singularity containers: 'Yes' - Output file: 'nf-core-rnaseq-3.0.tar.gz' + Pull containers: 'singularity' + Using $NXF_SINGULARITY_CACHEDIR': /path/to/demo/cachedir + Output directory: 'nf-core-rnaseq-3.0' INFO Downloading workflow files from GitHub INFO Downloading centralised configs from GitHub -INFO Fetching container names for workflow INFO Found 29 containers -INFO Tip: Set env var $NXF_SINGULARITY_CACHEDIR to use a central cache for container downloads Downloading singularity images ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% • 29/29 completed INFO Compressing download.. INFO Command to extract files: tar -xzf nf-core-rnaseq-3.0.tar.gz INFO MD5 checksum for nf-core-rnaseq-3.0.tar.gz: 9789a9e0bda50f444ab0ee69cc8a95ce ``` -The tool automatically compresses all of the resulting file in to a `.tar.gz` archive. -You can choose other formats (`.tar.bz2`, `zip`) or to not compress (`none`) with the `-c`/`--compress` flag. -The console output provides the command you need to extract the files. - -Once uncompressed, you will see something like the following file structure for the downloaded pipeline: +Once downloaded, you will see something like the following file structure for the downloaded pipeline: ```console -$ tree -L 2 nf-core-methylseq-1.4/ +$ tree -L 2 nf-core-rnaseq-3.0/ nf-core-rnaseq-3.0 ├── configs @@ -404,7 +416,11 @@ nf-core-rnaseq-3.0 └── main.nf ``` -You can run the pipeline by simply providing the directory path for the `workflow` folder to your `nextflow run` command. +You can run the pipeline by simply providing the directory path for the `workflow` folder to your `nextflow run` command: + +```bash +nextflow run /path/to/download/nf-core-rnaseq-3.0/workflow/ --input mydata.csv # usual parameters here +``` ### Downloaded nf-core configs @@ -414,7 +430,7 @@ So using `-profile ` should work if available within [nf-core/configs](htt ### Downloading singularity containers If you're using Singularity, the `nf-core download` command can also fetch the required Singularity container images for you. -To do this, specify the `--singularity` option. +To do this, select `singularity` in the prompt or specify `--container singularity` in the command. Your archive / target output directory will then include three folders: `workflow`, `configs` and also `singularity-containers`. The downloaded workflow files are again edited to add the following line to the end of the pipeline's `nextflow.config` file: @@ -433,7 +449,8 @@ We highly recommend setting the `$NXF_SINGULARITY_CACHEDIR` environment variable If found, the tool will fetch the Singularity images to this directory first before copying to the target output archive / directory. Any images previously fetched will be found there and copied directly - this includes images that may be shared with other pipelines or previous pipeline version downloads or download attempts. -If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose specify `--singularity-cache`. +If you are running the download on the same system where you will be running the pipeline (eg. a shared filesystem where Nextflow won't have an internet connection at a later date), you can choose to _only_ use the cache via a prompt or cli options `--singularity-cache-only` / `--singularity-cache-copy`. + This instructs `nf-core download` to fetch all Singularity images to the `$NXF_SINGULARITY_CACHEDIR` directory but does _not_ copy them to the workflow archive / directory. The workflow config file is _not_ edited. This means that when you later run the workflow, Nextflow will just use the cache folder directly. @@ -451,15 +468,15 @@ Where both are found, the download URL is preferred. Once a full list of containers is found, they are processed in the following order: -1. If the target image already exists, nothing is done (eg. with `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache` specified) -2. If found in `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache` is _not_ specified, they are copied to the output directory +1. If the target image already exists, nothing is done (eg. with `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache-only` specified) +2. If found in `$NXF_SINGULARITY_CACHEDIR` and `--singularity-cache-only` is _not_ specified, they are copied to the output directory 3. If they start with `http` they are downloaded directly within Python (default 4 at a time, you can customise this with `--parallel-downloads`) 4. If they look like a Docker image name, they are fetched using a `singularity pull` command * This requires Singularity to be installed on the system and is substantially slower Note that compressing many GBs of binary files can be slow, so specifying `--compress none` is recommended when downloading Singularity images. -If you really like hammering your internet connection, you can set `--parallel-downloads` to a large number to download loads of images at once. +If the download speeds are much slower than your internet connection is capable of, you can set `--parallel-downloads` to a large number to download loads of images at once. ## Pipeline software licences From 205340bd24c2a5c7650cd1d70ecd7b3a82592c40 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Thu, 6 May 2021 23:46:39 +0200 Subject: [PATCH 44/45] Download docs - didn't compress --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 670ffebd6..8e7b7206b 100644 --- a/README.md +++ b/README.md @@ -389,10 +389,7 @@ INFO Saving 'nf-core/rnaseq INFO Downloading workflow files from GitHub INFO Downloading centralised configs from GitHub INFO Found 29 containers -Downloading singularity images ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% • 29/29 completed -INFO Compressing download.. -INFO Command to extract files: tar -xzf nf-core-rnaseq-3.0.tar.gz -INFO MD5 checksum for nf-core-rnaseq-3.0.tar.gz: 9789a9e0bda50f444ab0ee69cc8a95ce +Downloading singularity images ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100% • 29/29 completed ``` Once downloaded, you will see something like the following file structure for the downloaded pipeline: From 19f2d5c1efe19d3cfc64a9ba0a41be84718c1118 Mon Sep 17 00:00:00 2001 From: Phil Ewels Date: Fri, 7 May 2021 20:21:33 +0200 Subject: [PATCH 45/45] Address review comments for singularty cachedir prompt --- nf_core/download.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/nf_core/download.py b/nf_core/download.py index 0ce7891ca..1e567d340 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -261,12 +261,11 @@ def prompt_use_singularity_cachedir(self): # Prompt user for a cache directory path cachedir_path = None while cachedir_path is None: - cachedir_path = os.path.abspath( - questionary.path( - "Specify the path:", only_directories=True, style=nf_core.utils.nfcore_question_style - ).unsafe_ask() - ) - if cachedir_path == os.path.abspath(""): + prompt_cachedir_path = questionary.path( + "Specify the path:", only_directories=True, style=nf_core.utils.nfcore_question_style + ).unsafe_ask() + cachedir_path = os.path.abspath(os.path.expanduser(prompt_cachedir_path)) + if prompt_cachedir_path == "": log.error(f"Not using [blue]$NXF_SINGULARITY_CACHEDIR[/]") cachedir_path = False elif not os.path.isdir(cachedir_path):