Add option to mangle alert words in retry warnings and ruff it

.github/workflows/python-formatting.yml

@@ -0,0 +1,10 @@
+name: check format using ruff
+on: [push]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1
+        with:
+          args: format --check

.github/workflows/python-linting.yml

@@ -0,0 +1,8 @@
+name: lint code using ruff
+on: [push]
+jobs:
+  ruff:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: chartboost/ruff-action@v1

.pre-commit-config.yaml

@@ -1,13 +1,9 @@
 repos:
--   repo: https://github.com/psf/black
-    rev: 23.7.0
-    hooks:
-    - id: black
-      language_version: python3.10
-
--   repo: https://github.com/pycqa/isort
-    rev: 5.12.0
-    hooks:
-    - id: isort
-      name: isort (python)
-      language_version: python3.10
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.2.1
+  hooks:
+    # Run the linter.
+    - id: ruff
+    # Run the formatter.
+    - id: ruff-format

pyrightconfig.json

@@ -0,0 +1,8 @@
+{
+    "ignore": [
+        "**/node_modules",
+        "**/__pycache__",
+        "**/*.ipynb",
+        ".git"
+    ]
+}

ruff.toml

@@ -0,0 +1,86 @@
+# Copied originally from pandas. This config requires ruff >= 0.2.
+target-version = "py310"
+
+# fix = true
+lint.unfixable = []
+
+lint.select = [
+  "I", # isort
+  "F", # pyflakes
+  "E", "W", # pycodestyle
+  "YTT", # flake8-2020
+  "B", # flake8-bugbear
+  "Q", # flake8-quotes
+  "T10", # flake8-debugger
+  "INT", # flake8-gettext
+  "PLC", "PLE", "PLR", "PLW", # pylint
+  "PIE", # misc lints
+  "PYI", # flake8-pyi
+  "TID", # tidy imports
+  "ISC", # implicit string concatenation
+  "TCH", # type-checking imports
+  "C4", # comprehensions
+  "PGH" # pygrep-hooks
+]
+
+# Some additional rules that are useful
+lint.extend-select = [
+"UP009",  # UTF-8 encoding declaration is unnecessary
+"SIM118",  # Use `key in dict` instead of `key in dict.keys()`
+"D205",  # One blank line required between summary line and description
+"ARG001",  # Unused function argument
+"RSE102",  # Unnecessary parentheses on raised exception
+"PERF401",  # Use a list comprehension to create a transformed list
+]
+
+lint.ignore = [
+  "ISC001", # Disable this for compatibility with ruff format
+  "B028", # No explicit `stacklevel` keyword argument found
+  "B905", # `zip()` without an explicit `strict=` parameter
+  "E402", # module level import not at top of file
+  "E731", # do not assign a lambda expression, use a def
+  "PLC1901", # compare-to-empty-string
+  "PLR0911", # Too many returns
+  "PLR0912", # Too many branches
+  "PLR0913", # Too many arguments to function call
+  "PLR0915", # Too many statements
+  "PLR2004", # Magic number
+]
+
+# TODO : fix these and stop ignoring. Commented out ones are common and OK to except.
+lint.extend-ignore = [
+  "PGH004", # Use specific rule codes when using `noqa`
+#  "C401", # Unnecessary generator (rewrite as a `set` comprehension)
+#  "C402", # Unnecessary generator (rewrite as a dict comprehension)
+#  "C405", # Unnecessary `list` literal (rewrite as a `set` literal)
+#  "C408", # Unnecessary `dict` call (rewrite as a literal)
+#  "C416", # Unnecessary `dict` comprehension (rewrite using `dict()`)
+#  "G010", # warn is deprecated in favor of warning
+#  "PYI056", # Calling `.append()` on `__all__` may not be supported by all type checkers
+    #
+    # These were noted in PR #57 with the transition to ruff and are being ignored only
+    # to minimize unrelated diffs there. These should mosty be fixed in a subsequent PR.
+    #
+    "ARG001",
+    "PYI041",
+    "D205",
+    "I001",
+    "C403",
+    "B904",
+    "B018",
+    "F541",
+]
+
+extend-exclude = [
+  "docs",
+]
+
+[lint.pycodestyle]
+max-line-length = 100 # E501 reports lines that exceed the length of 100.
+
+[lint.extend-per-file-ignores]
+"__init__.py" = ["E402", "F401", "F403"]
+# For tests:
+# - D205: Don't worry about test docstrings
+# - ARG001: Unused function argument false positives for some fixtures
+"**/tests/test_*.py" = ["D205", "ARG001", "E501"]

ska_helpers/retry/api.py

@@ -1,11 +1,13 @@
 import functools
-import logging
 import random
+import re
 import sys
 import time
 import traceback
 
-logging_logger = logging.getLogger(__name__)
+from ska_helpers.logging import basic_logger
+
+logging_logger = basic_logger(__name__, format="%(message)s")
 
 
 class RetryError(Exception):
@@ -21,6 +23,33 @@ def __init__(self, failures):
         self.failures = failures
 
 
+def _mangle_alert_words(msg):
+    """
+    Mangle alert words "warning", "error", "fatal", "fail", "exception" in a string.
+
+    This is done by replacing "i" or "l" with "1" and "o" with "0" in the middle of
+    any of these words. The intent is to avoid triggering the task schedule "check" for
+    for those words. This is done with a case-insensitive regex substitution.
+
+    Example::
+
+        >>> mangle_alert_words("WARNING: This is a fatal Error message.")
+        'WARN1NG: This is a fata1 Err0r message.'
+
+    :param msg: the string to mangle.
+    :returns: the mangled string.
+    """
+    for re_word, sub in (
+        ("(warn)(i)(ng)", "1"),
+        ("(err)(o)(r)", "0"),
+        ("(fata)(l)()", "1"),
+        ("(fai)(l)()", "1"),
+        ("(excepti)(o)(n)", "0"),
+    ):
+        msg = re.sub(re_word, rf"\g<1>{sub}\3", msg, flags=re.IGNORECASE)
+    return msg
+
+
 def __retry_internal(
     f,
     exceptions=Exception,
@@ -30,6 +59,7 @@ def __retry_internal(
     backoff=1,
     jitter=0,
     logger=logging_logger,
+    mangle_alert_words=False,
     args=None,
     kwargs=None,
 ):
@@ -46,6 +76,9 @@ def __retry_internal(
                    fixed if a number, random if a range tuple (min, max)
     :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
                    default: retry.logging_logger. if None, logging is disabled.
+    :param mangle_alert_words: if True, mangle alert words "warning", "error", "fatal",
+                   "exception", "fail" when issuing a logger warning message.
+                   Default: False.
     :param args: tuple, function args
     :param kwargs: dict, function kwargs
     :returns: the result of the f function.
@@ -76,10 +109,13 @@ def __retry_internal(
                 call_args_str = ", ".join(str(arg) for arg in call_args)
                 func_name = getattr(f, "__name__", "func")
                 func_call = f"{func_name}({call_args_str})"
-                logger.warning(
+                msg = (
                     f"WARNING: {func_call} exception: {e}, retrying "
                     f"in {_delay} seconds..."
                 )
+                if mangle_alert_words:
+                    msg = _mangle_alert_words(msg)
+                logger.warning(msg)
 
             time.sleep(_delay)
             _delay *= backoff
@@ -101,6 +137,7 @@ def retry(
     backoff=1,
     jitter=0,
     logger=logging_logger,
+    mangle_alert_words=False,
 ):
     """Returns a retry decorator.
 
@@ -113,6 +150,8 @@ def retry(
                    fixed if a number, random if a range tuple (min, max)
     :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
                    default: retry.logging_logger. if None, logging is disabled.
+    :param mangle_alert_words: if True, mangle alert words "warning", "error", "fatal",
+                   "exception" when issuing a logger warning message. Default: False.
     :returns: a retry decorator.
     """
 
@@ -128,6 +167,7 @@ def wrapper(*args, **kwargs):
                 backoff,
                 jitter,
                 logger,
+                mangle_alert_words=mangle_alert_words,
                 args=args,
                 kwargs=kwargs,
             )
@@ -148,6 +188,7 @@ def retry_call(
     backoff=1,
     jitter=0,
     logger=logging_logger,
+    mangle_alert_words=False,
 ):
     """
     Calls a function and re-executes it if it failed.
@@ -164,6 +205,9 @@ def retry_call(
                    fixed if a number, random if a range tuple (min, max)
     :param logger: logger.warning(fmt, error, delay) will be called on failed attempts.
                    default: retry.logging_logger. if None, logging is disabled.
+    :param mangle_alert_words: if True, mangle alert words "warning", "error", "fatal",
+                   "exception", "fail" when issuing a logger warning message.
+                   Default: False.
     :returns: the result of the f function.
     """
     if args is None:
@@ -180,6 +224,7 @@ def retry_call(
         backoff,
         jitter,
         logger,
+        mangle_alert_words=mangle_alert_words,
         args=args,
         kwargs=kwargs,
     )
@@ -192,21 +237,37 @@ def tables_open_file(*args, **kwargs):
     it will try again after 2 seconds and once more after 4 seconds.
 
     :param *args: args passed through to tables.open_file()
-    :param **kwargs: kwargs passed through to tables.open_file()
+    :param mangle_alert_words: (keyword-only) if True, mangle alert words "warning",
+                   "error", "fatal", "exception", "fail" when issuing a logger warning
+                   message. Default: True.
+    :param retry_delay: (keyword-only) initial delay between attempts. default: 2.
+    :param retry_tries: (keyword-only) the maximum number of attempts. default: 3.
+    :param retry_backoff: (keyword-only) multiplier applied to delay between attempts.
+                     default: 2.
+    :param retry_logger: (keyword-only) logger.warning(msg) will be called.
+    :param **kwargs: additional kwargs passed through to tables.open_file()
     :returns: tables file handle
     """
     import tables
     import tables.exceptions
 
     import ska_helpers.retry
 
+    mangle_alert_words = kwargs.pop("mangle_alert_words", True)
+    retry_delay = kwargs.pop("retry_delay", 2)
+    retry_tries = kwargs.pop("retry_tries", 3)
+    retry_backoff = kwargs.pop("retry_backoff", 2)
+    retry_logger = kwargs.pop("retry_logger", logging_logger)
+
     h5 = ska_helpers.retry.retry_call(
         tables.open_file,
         args=args,
         kwargs=kwargs,
         exceptions=tables.exceptions.HDF5ExtError,
-        delay=2,
-        tries=3,
-        backoff=2,
+        delay=retry_delay,
+        tries=retry_tries,
+        backoff=retry_backoff,
+        logger=retry_logger,
+        mangle_alert_words=mangle_alert_words,
     )
     return h5