Merge pull request #106 from cokelaer/main

v1.0

.github/workflows/main.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
       - dev
-  workflow_dispatch:      
+  workflow_dispatch:
   pull_request:
     branches-ignore: []
   schedule:

README.rst

@@ -276,15 +276,15 @@ of the sequana wrappers and will use the official sequana github repository by d
 (https://github.com/sequana/sequana-wrappers). This may be overwritten. For instance, you may use a local clone. To do
 so, you will need to create an environment variable::
 
-    export SEQUANA_WRAPPERS="git+file:///home/user/github/sequana-wrappers
+    export SEQUANA_WRAPPERS="git+file:///home/user/github/sequana-wrappers"
 
 If you decide to use singularity/apptainer, one common error on a cluster is that non-standard paths are not found. You can bind them using the -B option but a more general set up is to create thos environment variable::
 
-    export SINGULARITY_BINDPATH="  /path_to_bind"
+    export SINGULARITY_BINDPATH="/path_to_bind"
 
 for Singularity setup, or ::
 
-    export APPTAINER_BINDPATH=" /path_to_bind"
+    export APPTAINER_BINDPATH="/path_to_bind"
 
 for Apptainer setup.
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api"
 #maintainer ?#maintainer email
 [tool.poetry]
 name = "sequana_pipetools"
-version = "0.17.3"
+version = "1.0.0"
 description = "A set of tools to help building or using Sequana pipelines"
 authors = ["Sequana Team"]
 license = "BSD-3"
@@ -24,6 +24,7 @@ classifiers = [
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Topic :: Software Development :: Libraries :: Python Modules",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
         "Topic :: Scientific/Engineering :: Information Analysis",
@@ -34,7 +35,7 @@ packages = [
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4.0"
-easydev = ">=0.12.1"
+easydev = ">=0.12"
 parse = ">=1.19.0"
 "ruamel.yaml" = ">=0.18.5"
 packaging = ">=23.1"

sequana_pipetools/scripts/main.py

@@ -15,7 +15,6 @@
 import sys
 
 import rich_click as click
-from pkg_resources import DistributionNotFound
 
 from sequana_pipetools import version
 from sequana_pipetools.misc import url2hash
@@ -227,7 +226,7 @@ def main(**kwargs):
         try:
             c = ClickComplete(name)
             c.save_completion_script()
-        except DistributionNotFound:  # pragma: no cover
+        except Exception:  # pragma: no cover
             click.echo(f"# Warning {name} could not be imported. Nothing done")
         finally:
             click.echo("Please source the files using:: \n")

sequana_pipetools/sequana_manager.py

@@ -138,7 +138,10 @@ def __init__(self):
         )
 
         if self.options.apptainer_prefix:  # pragma: no cover
-            self.apptainer_prefix = self.options.apptainer_prefix
+            self.apptainer_prefix = Path(self.options.apptainer_prefix).resolve()
+            if self.apptainer_prefix.exists() is False:
+                logger.error(f"{self.apptainer_prefix} does not exist")
+                sys.exit(1)
             self.local_apptainers = False
         else:  # pragma: no cover
             self.apptainer_prefix = os.environ.get("SEQUANA_SINGULARITY_PREFIX", f"{self.workdir}/.sequana/apptainers")
@@ -218,12 +221,7 @@ def setup(self):
             if self.local_apptainers:
                 self.command += " --singularity-prefix .sequana/apptainers"
             else:
-                if Path(self.apptainer_prefix).is_absolute():
-                    self.command += f" --singularity-prefix {self.apptainer_prefix} "
-                else:
-                    # if we set prefix to e.g. ./images then in the pipeline/script.sh,
-                    # the prefix is also ./images whereas it should be ../images
-                    self.command += f" --singularity-prefix ../{self.apptainer_prefix} "
+                self.command += f" --singularity-prefix {self.apptainer_prefix} "
 
         # set up core/jobs options
         if self.options.profile == "local":

sequana_pipetools/snaketools/errors.py

@@ -23,6 +23,8 @@ def __init__(self, *args, **kwargs):
 
     def status(self, working_directory="./", logs_directory="logs"):
 
+        print("\n\u274C one or several errors were detected. Please check carefully the above message, or the logs/ directory (for HPC/cluster usage). In the later case, some hints may be provided here below. " )
+
         # we allows slurm to be detected even though we are not on a cluster
         # this allows users to debug slurm job through NFS mounting
         try:  # let us try to introspect the slurm files

sequana_pipetools/snaketools/pipeline_manager.py

@@ -139,8 +139,12 @@ def clean_multiqc(self, filename):
 
     def onerror(self):
         """Try to report error from slurm"""
-        p = PipeError(self.name)
-        p.status()
+        try:
+            p = PipeError(self.name)
+            p.status()
+            print(f"\nIf you encoutered an error using sequana_{self.name}, please copy paste the above message and create a New Issue on https://github.com/sequana/{self.name}/issues")
+        except Exception as err:
+            print
 
     def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], outdir="."):
         # add a Makefile
@@ -164,6 +168,11 @@ def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], outdir="."
                             version = "unknown"
                         fout.write(f"{dep}\t{version}\n")
 
+        if os.path.exists("summary.html"):
+            print("\u2705 Another successful analysis. Open summary.html in your browser. Have fun.")
+        else:
+            print("\u2705 Another successful analysis. Have fun.")
+
     def get_html_summary(self, float="left", width=30):
         import pandas as pd
 
@@ -252,7 +261,7 @@ class PipelineManager(PipelineManagerBase):
     a so-called read-tag to identify the first and second file. Traditionnally, e.g.,
     with illumina sequencers the read tag are _R1_ and _R2_ or a trailing _1 and _2
     Note that samples names have sometimes this tag included. Consider e.g.
-    sample_replicate_1_R1_.fastq.gz or sample_replicate_1_1.fastq.gz then you can imagine that
+    `sample_replicate_1_R1_.fastq.gz` or `sample_replicate_1_1.fastq.gz` then you can imagine that
     it is tricky to handle.
 
     The sample names are extracted by cutting filenames on the first dot that is encoutered

sequana_pipetools/snaketools/profile.py

@@ -1,12 +1,27 @@
 """ Setting up profile to setup """
-import importlib.resources as pkg_resources
+
+try:
+    import importlib.resources as resources
+except ImportError:  # pragma: no cover
+    # Try backported to PY<37 `importlib_resources`.
+    import importlib_resources as resources
+
+
 from pathlib import Path
 
 
 def create_profile(workdir: Path, profile: str, **kwargs) -> str:
     """Create profile config in working directory."""
-    with pkg_resources.path("sequana_pipetools.resources", f"{profile}.yaml") as slurm_file:
-        slurm_text = slurm_file.read_text().format(**kwargs)
+    try:
+        slurm_file = resources.files("sequana_pipetools.resources").joinpath(f"{profile}.yaml")
+        with open(slurm_file, "r") as fin:
+            slurm_text = fin.read()
+            slurl_text = slurm_text.format(**kwargs)
+    except AttributeError:
+        # python 3.8 support for back compatibility
+        with resources.path("sequana_pipetools.resources", f"{profile}.yaml") as slurm_file:
+            slurm_text = slurm_file.read_text().format(**kwargs)
+
     outfile = workdir / f".sequana/profile_{profile}" / "config.yaml"
     outfile.parent.mkdir(parents=True, exist_ok=True)
     outfile.write_text(slurm_text)

sequana_pipetools/snaketools/sequana_config.py

@@ -18,10 +18,10 @@
 from pykwalify.core import Core, CoreError, SchemaError
 
 try:
-    import importlib.resources as pkg_resources
+    import importlib.resources as resources
 except ImportError:  # pragma: no cover
     # Try backported to PY<37 `importlib_resources`.
-    import importlib_resources as pkg_resources
+    import importlib_resources as resources
 
 import colorlog
 
@@ -182,8 +182,12 @@ def check_config_with_schema(self, schemafile):
 
         """
         # add custom extensions
-        with pkg_resources.path("sequana_pipetools.resources", "ext.py") as ext_name:
+        try:
+            ext_name = resources.files("sequana_pipetools.resources").joinpath("ext.py")
             extensions = [str(ext_name)]
+        except AttributeError:
+            with resources.path("sequana_pipetools.resources", "ext.py") as ext_name:
+                extensions = [str(ext_name)]
 
         # causes issue with ruamel.yaml 0.12.13. Works for 0.15
         warnings.simplefilter("ignore", ruamel.yaml.error.UnsafeLoaderWarning)

sequana_pipetools/snaketools/slurm.py

@@ -89,7 +89,7 @@ def __repr__(self):
     def _report(self):
         N = len(self.errors)
         message = "#" * 33 + " DEBUG REPORT " + "#" * 33 + "\n\n"
-        message += f"The analysis reached {self.percent}. A total of {N} error(s) has been found.\n"
+        message += f"The analysis reached {self.percent}. A total of {N} known error(s) have been found.\n"
         if N > 0:
             message += f"Errors are comming from rule(s): {','.join(set([e['rule'] for e in self.errors]))}\n\n"
 

tests/test_manager.py

@@ -7,8 +7,6 @@
 from sequana_pipetools import SequanaConfig
 #from sequana_pipetools.sequana_manager import get_pipeline_location
 
-import pkg_resources
-from packaging.version import parse as parse_version
 
 
 from . import test_dir
@@ -164,12 +162,7 @@ def test_pipeline_parse_containers(tmpdir):
     pm = SequanaManager(dd, "fastqc")
     # fastqc uses 3 apptainers:
 
-    fastqc_version = pkg_resources.get_distribution("sequana_fastqc").version
-
-    if parse_version(fastqc_version) >= parse_version("1.6.0"):
-        assert len(pm._get_section_content(pm.module.snakefile, "container:")) in [2, 3,4]
-    else:
-        assert len(pm._get_section_content(pm.module.snakefile, "container:")) == 0
+    len(pm._get_section_content(pm.module.snakefile, "container:")) in [2, 3,4]
 
 
 def test_multiple_downloads(tmpdir):