Tower UI no longer shows revisions for downloaded pipelines.

nf_core/download.py

@@ -22,12 +22,17 @@
 import rich
 import rich.progress
 from git.exc import GitCommandError, InvalidGitRepositoryError
+from pkg_resources import parse_version as VersionParser
 
 import nf_core
 import nf_core.list
 import nf_core.utils
 from nf_core.synced_repo import RemoteProgressbar, SyncedRepo
-from nf_core.utils import NFCORE_CACHE_DIR, NFCORE_DIR
+from nf_core.utils import (
+    NFCORE_CACHE_DIR,
+    NFCORE_DIR,
+    SingularityCacheFilePathValidator,
+)
 
 log = logging.getLogger(__name__)
 stderr = rich.console.Console(
@@ -157,7 +162,7 @@ def download_workflow(self):
             sys.exit(1)
 
         summary_log = [
-            f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',...,['+str(len(self.revision)-2)+' more revisions],...,'+self.revision[-1]}'",
+            f"Pipeline revision: '{', '.join(self.revision) if len(self.revision) < 5 else self.revision[0]+',['+str(len(self.revision)-2)+' more revisions],'+self.revision[-1]}'",
             f"Pull containers: '{self.container}'",
         ]
         if self.container == "singularity" and os.environ.get("NXF_SINGULARITY_CACHEDIR") is not None:
@@ -493,8 +498,8 @@ def prompt_singularity_cachedir_remote(self):
             cachedir_index = None
             while cachedir_index is None:
                 prompt_cachedir_index = questionary.path(
-                    "Specify a list of the remote images already present in the remote system :",
-                    file_filter="*.txt",
+                    "Specify a list of the container images that are already present on the remote system:",
+                    validate=SingularityCacheFilePathValidator,
                     style=nf_core.utils.nfcore_question_style,
                 ).unsafe_ask()
                 cachedir_index = os.path.abspath(os.path.expanduser(prompt_cachedir_index))
@@ -531,7 +536,8 @@ def read_remote_containers(self):
             except (FileNotFoundError, LookupError) as e:
                 log.error(f"[red]Issue with reading the specified remote $NXF_SINGULARITY_CACHE index:[/]\n{e}\n")
                 if stderr.is_interactive and rich.prompt.Confirm.ask(f"[blue]Specify a new index file and try again?"):
-                    self.prompt_singularity_cachedir_remote(retry=True)
+                    self.singularity_cache_index = None  # reset chosen path to index file.
+                    self.prompt_singularity_cachedir_remote()
                 else:
                     log.info("Proceeding without consideration of the remote $NXF_SINGULARITY_CACHE index.")
                     self.singularity_cache_index = None
@@ -731,7 +737,7 @@ def find_container_images(self, workflow_directory):
                                     r"(?<=container)[^\${}]+\${([^{}]+)}(?![^{]*})", contents
                                 )
 
-                                if bool(container_definition) & bool(container_definition.group(1)):
+                                if bool(container_definition) and bool(container_definition.group(1)):
                                     pattern = re.escape(container_definition.group(1))
                                     # extract the quoted string(s) following the variable assignment
                                     container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, contents)
@@ -880,7 +886,8 @@ def get_singularity_images(self, current_revision=""):
                             # Raise exception if this is not possible
                             log.error("Not able to pull image. Service might be down or internet connection is dead.")
                             raise r
-                        progress.update(task, advance=1)
+                        finally:
+                            progress.update(task, advance=1)
 
     def singularity_image_filenames(self, container):
         """Check Singularity cache for image, copy to destination folder if found.
@@ -1048,9 +1055,12 @@ def singularity_pull_image(self, container, out_path, cache_path, progress):
         if lines:
             # something went wrong with the container retrieval
             if any("FATAL: " in line for line in lines):
-                log.info("Singularity container retrieval failed with the following error:")
-                log.info("".join(lines))
-                raise FileNotFoundError(f'The container "{container}" is unavailable.\n{"".join(lines)}')
+                log.error(f'[bold red]The singularity image "{container}" could not be pulled:[/]\n\n{"".join(lines)}')
+                log.error(
+                    f'Skipping failed pull of "{container}". Please troubleshoot the command \n"{" ".join(singularity_command)}"\n\n\n'
+                )
+                progress.remove_task(task)
+                return
 
         # Copy cached download if we are using the cache
         if cache_path:
@@ -1252,8 +1262,8 @@ def tidy_tags_and_branches(self):
             desired_revisions = set(self.revision)
 
             # determine what needs pruning
-            tags_to_remove = {tag for tag in self.repo.tags if tag.name not in desired_revisions}
-            heads_to_remove = {head for head in self.repo.heads if head.name not in desired_revisions}
+            tags_to_remove = {tag for tag in self.repo.tags if tag.name not in desired_revisions.union({"latest"})}
+            heads_to_remove = {head for head in self.repo.heads if head.name not in desired_revisions.union({"latest"})}
 
             try:
                 # delete unwanted tags from repository
@@ -1268,9 +1278,34 @@ def tidy_tags_and_branches(self):
                 for head in heads_to_remove:
                     self.repo.delete_head(head)
 
-                # ensure all desired branches are available
+                # ensure all desired revisions/branches are available
                 for revision in desired_revisions:
-                    self.checkout(revision)
+                    if not self.repo.is_valid_object(revision):
+                        self.checkout(revision)
+                        self.repo.create_head(revision, revision)
+                        if self.repo.head.is_detached:
+                            self.repo.head.reset(index=True, working_tree=True)
+
+                # no branch exists, but one is required for Tower's UI to display revisions correctly). Thus, "latest" will be created.
+                if not bool(self.repo.heads):
+                    if self.repo.is_valid_object("latest"):
+                        # "latest" exists as tag but not as branch
+                        self.repo.create_head("latest", "latest")  # create a new head for latest
+                        self.checkout("latest")
+                    else:
+                        # desired revisions may contain arbitrary branch names that do not correspond to valid sematic versioning patterns.
+                        valid_versions = [
+                            VersionParser(v)
+                            for v in desired_revisions
+                            if re.match(r"\d+\.\d+(?:\.\d+)*(?:[\w\-_])*", v)
+                        ]
+                        # valid versions sorted in ascending order, last will be aliased as "latest".
+                        latest = sorted(valid_versions)[-1]
+                        self.repo.create_head("latest", latest)
+                        self.checkout(latest)
+                    if self.repo.head.is_detached:
+                        self.repo.head.reset(index=True, working_tree=True)
+
                 self.heads = self.repo.heads
 
                 # get all tags and available remote_branches

nf_core/utils.py

@@ -880,6 +880,23 @@ def prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False):
         )
 
 
+class SingularityCacheFilePathValidator(questionary.Validator):
+    """
+    Validator for file path specified as --singularity-cache-index argument in nf-core download
+    """
+
+    def validate(self, value):
+        if len(value.text):
+            if os.path.isfile(value.text):
+                return True
+            else:
+                raise questionary.ValidationError(
+                    message="Invalid remote cache index file", cursor_position=len(value.text)
+                )
+        else:
+            return True
+
+
 def get_repo_releases_branches(pipeline, wfs):
     """Fetches details of a nf-core workflow to download.
 

tests/test_download.py

@@ -261,8 +261,14 @@ def test_download_workflow_for_tower(self, tmp_dir):
         assert download_obj.workflow_repo
         assert isinstance(download_obj.workflow_repo, WorkflowRepo)
         assert issubclass(type(download_obj.workflow_repo), SyncedRepo)
+
         # corroborate that the other revisions are inaccessible to the user.
-        assert len(download_obj.workflow_repo.tags) == len(download_obj.revision)
+        all_tags = {tag.name for tag in download_obj.workflow_repo.tags}
+        all_heads = {head.name for head in download_obj.workflow_repo.heads}
+
+        assert set(download_obj.revision) == all_tags
+        # assert that the download has a "latest" branch.
+        assert "latest" in all_heads
 
         # download_obj.download_workflow_tower(location=tmp_dir) will run container image detection for all requested revisions
         assert isinstance(download_obj.containers, list) and len(download_obj.containers) == 33