diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4148945..1a47fb6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -5,7 +5,7 @@ on: branches: - main - dev - workflow_dispatch: + workflow_dispatch: pull_request: branches-ignore: [] schedule: diff --git a/README.rst b/README.rst index c4314e6..b7bc242 100644 --- a/README.rst +++ b/README.rst @@ -276,15 +276,15 @@ of the sequana wrappers and will use the official sequana github repository by d (https://github.com/sequana/sequana-wrappers). This may be overwritten. For instance, you may use a local clone. To do so, you will need to create an environment variable:: - export SEQUANA_WRAPPERS="git+file:///home/user/github/sequana-wrappers + export SEQUANA_WRAPPERS="git+file:///home/user/github/sequana-wrappers" If you decide to use singularity/apptainer, one common error on a cluster is that non-standard paths are not found. You can bind them using the -B option but a more general set up is to create thos environment variable:: - export SINGULARITY_BINDPATH=" /path_to_bind" + export SINGULARITY_BINDPATH="/path_to_bind" for Singularity setup, or :: - export APPTAINER_BINDPATH=" /path_to_bind" + export APPTAINER_BINDPATH="/path_to_bind" for Apptainer setup. diff --git a/pyproject.toml b/pyproject.toml index 0bd7a61..ae715c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "poetry.core.masonry.api" #maintainer ?#maintainer email [tool.poetry] name = "sequana_pipetools" -version = "0.17.3" +version = "1.0.0" description = "A set of tools to help building or using Sequana pipelines" authors = ["Sequana Team"] license = "BSD-3" @@ -24,6 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Scientific/Engineering :: Bio-Informatics", "Topic :: Scientific/Engineering :: Information Analysis", @@ -34,7 +35,7 @@ packages = [ [tool.poetry.dependencies] python = ">=3.8,<4.0" -easydev = ">=0.12.1" +easydev = ">=0.12" parse = ">=1.19.0" "ruamel.yaml" = ">=0.18.5" packaging = ">=23.1" diff --git a/sequana_pipetools/scripts/main.py b/sequana_pipetools/scripts/main.py index 1c948d0..4e56039 100644 --- a/sequana_pipetools/scripts/main.py +++ b/sequana_pipetools/scripts/main.py @@ -15,7 +15,6 @@ import sys import rich_click as click -from pkg_resources import DistributionNotFound from sequana_pipetools import version from sequana_pipetools.misc import url2hash @@ -227,7 +226,7 @@ def main(**kwargs): try: c = ClickComplete(name) c.save_completion_script() - except DistributionNotFound: # pragma: no cover + except Exception: # pragma: no cover click.echo(f"# Warning {name} could not be imported. Nothing done") finally: click.echo("Please source the files using:: \n") diff --git a/sequana_pipetools/sequana_manager.py b/sequana_pipetools/sequana_manager.py index f274176..fdbe7f5 100644 --- a/sequana_pipetools/sequana_manager.py +++ b/sequana_pipetools/sequana_manager.py @@ -138,7 +138,10 @@ def __init__(self): ) if self.options.apptainer_prefix: # pragma: no cover - self.apptainer_prefix = self.options.apptainer_prefix + self.apptainer_prefix = Path(self.options.apptainer_prefix).resolve() + if self.apptainer_prefix.exists() is False: + logger.error(f"{self.apptainer_prefix} does not exist") + sys.exit(1) self.local_apptainers = False else: # pragma: no cover self.apptainer_prefix = os.environ.get("SEQUANA_SINGULARITY_PREFIX", f"{self.workdir}/.sequana/apptainers") @@ -218,12 +221,7 @@ def setup(self): if self.local_apptainers: self.command += " --singularity-prefix .sequana/apptainers" else: - if Path(self.apptainer_prefix).is_absolute(): - self.command += f" --singularity-prefix {self.apptainer_prefix} " - else: - # if we set prefix to e.g. ./images then in the pipeline/script.sh, - # the prefix is also ./images whereas it should be ../images - self.command += f" --singularity-prefix ../{self.apptainer_prefix} " + self.command += f" --singularity-prefix {self.apptainer_prefix} " # set up core/jobs options if self.options.profile == "local": diff --git a/sequana_pipetools/snaketools/errors.py b/sequana_pipetools/snaketools/errors.py index d6df927..b080270 100644 --- a/sequana_pipetools/snaketools/errors.py +++ b/sequana_pipetools/snaketools/errors.py @@ -23,6 +23,8 @@ def __init__(self, *args, **kwargs): def status(self, working_directory="./", logs_directory="logs"): + print("\n\u274C one or several errors were detected. Please check carefully the above message, or the logs/ directory (for HPC/cluster usage). In the later case, some hints may be provided here below. " ) + # we allows slurm to be detected even though we are not on a cluster # this allows users to debug slurm job through NFS mounting try: # let us try to introspect the slurm files diff --git a/sequana_pipetools/snaketools/pipeline_manager.py b/sequana_pipetools/snaketools/pipeline_manager.py index 1762927..a559ea4 100644 --- a/sequana_pipetools/snaketools/pipeline_manager.py +++ b/sequana_pipetools/snaketools/pipeline_manager.py @@ -139,8 +139,12 @@ def clean_multiqc(self, filename): def onerror(self): """Try to report error from slurm""" - p = PipeError(self.name) - p.status() + try: + p = PipeError(self.name) + p.status() + print(f"\nIf you encoutered an error using sequana_{self.name}, please copy paste the above message and create a New Issue on https://github.com/sequana/{self.name}/issues") + except Exception as err: + print def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], outdir="."): # add a Makefile @@ -164,6 +168,11 @@ def teardown(self, extra_dirs_to_remove=[], extra_files_to_remove=[], outdir="." version = "unknown" fout.write(f"{dep}\t{version}\n") + if os.path.exists("summary.html"): + print("\u2705 Another successful analysis. Open summary.html in your browser. Have fun.") + else: + print("\u2705 Another successful analysis. Have fun.") + def get_html_summary(self, float="left", width=30): import pandas as pd @@ -252,7 +261,7 @@ class PipelineManager(PipelineManagerBase): a so-called read-tag to identify the first and second file. Traditionnally, e.g., with illumina sequencers the read tag are _R1_ and _R2_ or a trailing _1 and _2 Note that samples names have sometimes this tag included. Consider e.g. - sample_replicate_1_R1_.fastq.gz or sample_replicate_1_1.fastq.gz then you can imagine that + `sample_replicate_1_R1_.fastq.gz` or `sample_replicate_1_1.fastq.gz` then you can imagine that it is tricky to handle. The sample names are extracted by cutting filenames on the first dot that is encoutered diff --git a/sequana_pipetools/snaketools/profile.py b/sequana_pipetools/snaketools/profile.py index 32e59a9..fe549d3 100644 --- a/sequana_pipetools/snaketools/profile.py +++ b/sequana_pipetools/snaketools/profile.py @@ -1,12 +1,27 @@ """ Setting up profile to setup """ -import importlib.resources as pkg_resources + +try: + import importlib.resources as resources +except ImportError: # pragma: no cover + # Try backported to PY<37 `importlib_resources`. + import importlib_resources as resources + + from pathlib import Path def create_profile(workdir: Path, profile: str, **kwargs) -> str: """Create profile config in working directory.""" - with pkg_resources.path("sequana_pipetools.resources", f"{profile}.yaml") as slurm_file: - slurm_text = slurm_file.read_text().format(**kwargs) + try: + slurm_file = resources.files("sequana_pipetools.resources").joinpath(f"{profile}.yaml") + with open(slurm_file, "r") as fin: + slurm_text = fin.read() + slurl_text = slurm_text.format(**kwargs) + except AttributeError: + # python 3.8 support for back compatibility + with resources.path("sequana_pipetools.resources", f"{profile}.yaml") as slurm_file: + slurm_text = slurm_file.read_text().format(**kwargs) + outfile = workdir / f".sequana/profile_{profile}" / "config.yaml" outfile.parent.mkdir(parents=True, exist_ok=True) outfile.write_text(slurm_text) diff --git a/sequana_pipetools/snaketools/sequana_config.py b/sequana_pipetools/snaketools/sequana_config.py index 805d89c..5f6e2ea 100644 --- a/sequana_pipetools/snaketools/sequana_config.py +++ b/sequana_pipetools/snaketools/sequana_config.py @@ -18,10 +18,10 @@ from pykwalify.core import Core, CoreError, SchemaError try: - import importlib.resources as pkg_resources + import importlib.resources as resources except ImportError: # pragma: no cover # Try backported to PY<37 `importlib_resources`. - import importlib_resources as pkg_resources + import importlib_resources as resources import colorlog @@ -182,8 +182,12 @@ def check_config_with_schema(self, schemafile): """ # add custom extensions - with pkg_resources.path("sequana_pipetools.resources", "ext.py") as ext_name: + try: + ext_name = resources.files("sequana_pipetools.resources").joinpath("ext.py") extensions = [str(ext_name)] + except AttributeError: + with resources.path("sequana_pipetools.resources", "ext.py") as ext_name: + extensions = [str(ext_name)] # causes issue with ruamel.yaml 0.12.13. Works for 0.15 warnings.simplefilter("ignore", ruamel.yaml.error.UnsafeLoaderWarning) diff --git a/sequana_pipetools/snaketools/slurm.py b/sequana_pipetools/snaketools/slurm.py index 1b8e263..9416633 100644 --- a/sequana_pipetools/snaketools/slurm.py +++ b/sequana_pipetools/snaketools/slurm.py @@ -89,7 +89,7 @@ def __repr__(self): def _report(self): N = len(self.errors) message = "#" * 33 + " DEBUG REPORT " + "#" * 33 + "\n\n" - message += f"The analysis reached {self.percent}. A total of {N} error(s) has been found.\n" + message += f"The analysis reached {self.percent}. A total of {N} known error(s) have been found.\n" if N > 0: message += f"Errors are comming from rule(s): {','.join(set([e['rule'] for e in self.errors]))}\n\n" diff --git a/tests/test_manager.py b/tests/test_manager.py index ef90d41..5fc296a 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -7,8 +7,6 @@ from sequana_pipetools import SequanaConfig #from sequana_pipetools.sequana_manager import get_pipeline_location -import pkg_resources -from packaging.version import parse as parse_version from . import test_dir @@ -164,12 +162,7 @@ def test_pipeline_parse_containers(tmpdir): pm = SequanaManager(dd, "fastqc") # fastqc uses 3 apptainers: - fastqc_version = pkg_resources.get_distribution("sequana_fastqc").version - - if parse_version(fastqc_version) >= parse_version("1.6.0"): - assert len(pm._get_section_content(pm.module.snakefile, "container:")) in [2, 3,4] - else: - assert len(pm._get_section_content(pm.module.snakefile, "container:")) == 0 + len(pm._get_section_content(pm.module.snakefile, "container:")) in [2, 3,4] def test_multiple_downloads(tmpdir):