tests/ci: limit waved failed model loading

.github/assistant.py

@@ -132,11 +132,23 @@ def changed_domains(
         if not files:
             logging.debug("Only integrations was changed so not reason for deep testing...")
             return _return_empty
+
         # filter only docs files
         files_docs = [fn for fn in files if fn.startswith("docs")]
         if len(files) == len(files_docs):
             logging.debug("Only docs was changed so not reason for deep testing...")
             return _return_empty
+
+        files_markdown = [fn for fn in files if fn.endswith(".md")]
+        if len(files) == len(files_markdown):
+            logging.debug("Only markdown files was changed so not reason for deep testing...")
+            return _return_empty
+
+        files_testing = [fn for fn in files if fn.startswith("tests") and "test_" not in fn]
+        if files_testing:
+            logging.debug("Some testing files was changed -> rather test everything...")
+            return _return_all
+
         # files in requirements folder
         files_req = [fn for fn in files if fn.startswith("requirements")]
         req_domains = [fn.split("/")[1] for fn in files_req]
@@ -147,19 +159,21 @@ def changed_domains(
             return _return_all
 
         # filter only package files and skip inits
-        _is_in_test = lambda fn: fn.startswith("tests")
-        _filter_pkg = lambda fn: _is_in_test(fn) or (fn.startswith("src/torchmetrics") and "__init__.py" not in fn)
+        _is_in_test = lambda fname: fname.startswith("tests")
+        _filter_pkg = lambda fname: _is_in_test(fname) or (
+            fname.startswith("src/torchmetrics") and "__init__.py" not in fname
+        )
         files_pkg = [fn for fn in files if _filter_pkg(fn)]
         if not files_pkg:
             return _return_all
 
         # parse domains
-        def _crop_path(fname: str, paths: list[str]) -> str:
+        def _crop_path(fname: str, paths: tuple[str] = ("src/torchmetrics/", "tests/unittests/", "functional/")) -> str:
             for p in paths:
                 fname = fname.replace(p, "")
             return fname
 
-        files_pkg = [_crop_path(fn, ["src/torchmetrics/", "tests/unittests/", "functional/"]) for fn in files_pkg]
+        files_pkg = [_crop_path(fn) for fn in files_pkg]
         # filter domain names
         tm_modules = [fn.split("/")[0] for fn in files_pkg if "/" in fn]
         # filter general (used everywhere) sub-packages

.github/workflows/ci-integrate.yml

@@ -36,7 +36,7 @@ jobs:
           - { python-version: "3.10", requires: "latest", os: "ubuntu-22.04" }
           # - { python-version: "3.10", requires: "latest", os: "macOS-14" } # M1 machine # todo: crashing for MPS out of memory
     env:
-      PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+      PYTORCH_URL: "http://download.pytorch.org/whl/cpu/"
       FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
       PYPI_CACHE: "_ci-cache_PyPI"
 
@@ -64,7 +64,7 @@ jobs:
           # this was updated in `source cashing` by optional oldest
           cat requirements/_integrate.txt
           # to have install pyTorch
-          pip install -e . "setuptools==69.5.1" --find-links=${PYTORCH_URL}
+          pip install -e . "setuptools==69.5.1" --extra-index-url=${PYTORCH_URL}
 
           # adjust version to PT ecosystem based on installed TM
           python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
@@ -73,7 +73,7 @@ jobs:
 
           # install package and dependencies
           pip install -e . -r requirements/_tests.txt -r requirements/_integrate.txt \
-            --find-links=${PYTORCH_URL} --find-links=${PYPI_CACHE} \
+            --extra-index-url="${PYTORCH_URL}" --find-links="${PYPI_CACHE}" \
             --upgrade-strategy eager
           pip list
 

.github/workflows/ci-tests.yml

@@ -64,7 +64,7 @@ jobs:
       FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
       TOKENIZERS_PARALLELISM: false
       TEST_DIRS: ${{ needs.check-diff.outputs.test-dirs }}
-      PIP_EXTRA_INDEX_URL: "--find-links=https://download.pytorch.org/whl/cpu/torch_stable.html"
+      PIP_EXTRA_INDEX_URL: "--extra-index-url=http://download.pytorch.org/whl/cpu/"
       UNITTEST_TIMEOUT: "" # by default, it is not set
 
     # Timeout: https://stackoverflow.com/a/59076067/4521646
@@ -105,7 +105,7 @@ jobs:
 
       - name: Switch to PT test URL
         if: ${{ matrix.pytorch-version == '2.6.0' }}
-        run: echo 'PIP_EXTRA_INDEX_URL=--extra-index-url https://download.pytorch.org/whl/test/cpu/' >> $GITHUB_ENV
+        run: echo 'PIP_EXTRA_INDEX_URL=--extra-index-url="https://download.pytorch.org/whl/test/cpu/"' >> $GITHUB_ENV
       - name: Install pkg
         timeout-minutes: 25
         run: |

.github/workflows/docs-build.yml

@@ -18,7 +18,7 @@ defaults:
 
 env:
   FREEZE_REQUIREMENTS: "1"
-  TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+  TORCH_URL: "http://download.pytorch.org/whl/cpu/"
   PYPI_CACHE: "_ci-cache_PyPI"
   PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: "python"
   TOKENIZERS_PARALLELISM: false
@@ -39,7 +39,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.9"
+          python-version: "3.x"
 
       - name: source cashing
         uses: ./.github/actions/pull-caches
@@ -52,7 +52,7 @@ jobs:
           make get-sphinx-template
           # install with -e so the path to source link comes from this project not from the installed package
           pip install -e . -U -r requirements/_docs.txt \
-            --find-links="${PYPI_CACHE}" --find-links="${TORCH_URL}"
+            --find-links="${PYPI_CACHE}" --extra-index-url="${TORCH_URL}"
       - run: pip list
       - name: Full build for deployment
         if: github.event_name != 'pull_request'

tests/unittests/_helpers/wrappers.py

@@ -9,7 +9,8 @@
 _ERROR_CONNECTION_MESSAGE_PATTERNS = (
     "We couldn't connect to",
     "Connection error",
-    "Can't load",
+    # "Can't load",  # fixme: this hid breaking change in transformers, so make it more specific
+    # "`nltk` resource `punkt` is",  # todo: this is not intuitive ahy this is a connection issue
 )