Merge branch 'master' into dependabot-pip-requirements-transformers-g…

…t-4.4.0-and-lt-4.48.0

.github/actions/pull-caches/action.yml

@@ -18,6 +18,14 @@ inputs:
     description: cache restore/dump key
     required: false
     default: "pypi-packages"
+  cache-torch-HF:
+    description: "cache torch and HF"
+    required: false
+    default: "true"
+  cache-references:
+    description: "cache metrics references"
+    required: false
+    default: "false"
 
 runs:
   using: "composite"
@@ -67,6 +75,7 @@ runs:
       shell: bash
 
     - name: Cache Torch & HF
+      if: inputs.cache-torch-HF == 'true' # since the input is string
       continue-on-error: true
       uses: actions/cache/restore@v3
       with:
@@ -75,6 +84,7 @@ runs:
         key: ci-caches
 
     - name: Restored Torch & HF
+      if: inputs.cache-torch-HF == 'true' # since the input is string
       run: |
         mkdir -p $CACHES_DIR
         pip install -q py-tree
@@ -83,14 +93,15 @@ runs:
 
     - name: Cache References
       # do not use this cache for dispatch and crone, to enable rebuild caches if needed
-      if: github.event_name != 'workflow_dispatch' && github.event_name != 'schedule'
+      if: github.event_name != 'workflow_dispatch' && github.event_name != 'schedule' && inputs.cache-references == 'true'
       continue-on-error: true
       uses: actions/cache/restore@v3
       with:
         path: tests/_cache-references
         key: cache-references
 
     - name: Restored References
+      if: inputs.cache-references == 'true' # since the input is string
       continue-on-error: true
       working-directory: tests/
       run: |

.github/actions/push-caches/action.yml

@@ -14,6 +14,18 @@ inputs:
     description: location to pull PyTorch from
     required: false
     default: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+  cache-artifact-appendix:
+    description: "unique name or running index"
+    required: false
+    default: ""
+  cache-torch-HF:
+    description: "cache torch and HF"
+    required: false
+    default: "true"
+  cache-references:
+    description: "cache metrics references"
+    required: false
+    default: "false"
 
 runs:
   using: "composite"
@@ -23,66 +35,50 @@ runs:
       shell: bash
 
     - name: Freeze local emv.
+      if: inputs.cache-artifact-appendix != ''
       run: |
         pip freeze > requirements.dump
         cat requirements.dump
       shell: bash
 
-    #- name: Filter self pkg
-    #  run: |
-    #    import os
-    #    fp = 'requirements.dump'
-    #    with open(fp) as fopen:
-    #      lines = [ln.strip() for ln in fopen.readlines()]
-    #    lines = [ln.split('+')[0] for ln in lines if '-e ' not in ln]
-    #    with open(fp, 'w') as fwrite:
-    #      fwrite.writelines([ln + os.linesep for ln in lines])
-    #  shell: python
-
     - name: Dump wheels
+      if: inputs.cache-artifact-appendix != ''
       run: |
         pip wheel -r requirements/_devel.txt --prefer-binary \
-          --wheel-dir=.pip-wheels \
+          --wheel-dir=_pip-wheels \
           -f ${{ inputs.torch-url }} -f ${{ inputs.pypi-dir }}
-        ls -lh .pip-wheels
+        ls -lh _pip-wheels
       shell: bash
 
-    - name: Cache pull packages
-      uses: actions/cache/restore@v3
-      with:
-        enableCrossOsArchive: true
-        path: ${{ inputs.pypi-dir }}
-        key: ${{ inputs.pypi-key }}
-
-    - name: Find diff
-      id: wheels-diff
+    - name: Move new packages to staging
+      if: inputs.cache-artifact-appendix != ''
       run: |
-        import os, glob
-        wheels = [os.path.basename(p) for p in glob.glob(".pip-wheels/*")]
-        pkgs = [os.path.basename(p) for p in glob.glob("${{ inputs.pypi-dir }}/*")]
-        diff = [w for w in wheels if w not in pkgs]
-        print(diff)
-        with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
-            print(f'count-new={len(diff)}', file=fh)
-      shell: python
-
-    - run: cp .pip-wheels/* ${{ inputs.pypi-dir }}
-      if: ${{ steps.wheels-diff.outputs.count-new != 0 }}
+        mkdir -p _pip-staging
+        python .github/assistant.py move_new_packages \
+          --dir-cache="${{ inputs.pypi-dir }}" \
+          --dir_local="_pip-wheels" \
+          --dir_staging="_pip-staging"
+        ls -lh _pip-staging/
+        # count files in the staging dir
+        file_count=$(ls -1 "_pip-staging/" | wc -l)
+        echo "NUM_PACKAGES=$file_count" >> $GITHUB_ENV
       shell: bash
 
-    - name: Cache push packages
-      if: ${{ steps.wheels-diff.outputs.count-new != 0 }}
-      uses: actions/cache/save@v3
+    - name: Upload new packages
+      if: inputs.cache-artifact-appendix != '' && env.NUM_PACKAGES != 0
+      uses: actions/upload-artifact@v4
       with:
-        enableCrossOsArchive: true
-        path: ${{ inputs.pypi-dir }}
-        key: ${{ inputs.pypi-key }}
+        name: ${{ inputs.pypi-key }}-run-${{ inputs.cache-artifact-appendix }}
+        path: _pip-staging
+        retention-days: 1
 
     - name: Post Torch & HF
+      if: inputs.cache-torch-HF == 'true' # since the input is string
       run: py-tree $CACHES_DIR
       shell: bash
 
     - name: Cache Torch & HF
+      if: inputs.cache-torch-HF == 'true' # since the input is string
       continue-on-error: true
       uses: actions/cache/save@v3
       with:
@@ -91,13 +87,16 @@ runs:
         key: ci-caches
 
     - name: Cache references
+      if: inputs.cache-references == 'true' # since the input is string
       continue-on-error: true
       uses: actions/cache/save@v3
       with:
         #enableCrossOsArchive: true
         path: tests/_cache-references
         key: cache-references
 
-    - name: Post References
-      run: py-tree tests/_cache-references/ --show_hidden
-      shell: bash
+    #- name: Post References
+    #  # This print taken soo many lines, so it is commented out
+    #  if: inputs.cache-references == 'true' # since the input is string
+    #  run: py-tree tests/_cache-references/ --show_hidden
+    #  shell: bash

.github/assistant.py

@@ -132,11 +132,24 @@ def changed_domains(
         if not files:
             logging.debug("Only integrations was changed so not reason for deep testing...")
             return _return_empty
+
         # filter only docs files
         files_docs = [fn for fn in files if fn.startswith("docs")]
         if len(files) == len(files_docs):
             logging.debug("Only docs was changed so not reason for deep testing...")
             return _return_empty
+
+        files_markdown = [fn for fn in files if fn.endswith(".md")]
+        if len(files) == len(files_markdown):
+            logging.debug("Only markdown files was changed so not reason for deep testing...")
+            return _return_empty
+
+        # filter only testing files which are not specific tests so for example configurations or helper tools
+        files_testing = [fn for fn in files if fn.startswith("tests") and not fn.endswith(".md") and "test_" not in fn]
+        if files_testing:
+            logging.debug("Some testing files was changed -> rather test everything...")
+            return _return_all
+
         # files in requirements folder
         files_req = [fn for fn in files if fn.startswith("requirements")]
         req_domains = [fn.split("/")[1] for fn in files_req]
@@ -147,19 +160,21 @@ def changed_domains(
             return _return_all
 
         # filter only package files and skip inits
-        _is_in_test = lambda fn: fn.startswith("tests")
-        _filter_pkg = lambda fn: _is_in_test(fn) or (fn.startswith("src/torchmetrics") and "__init__.py" not in fn)
+        _is_in_test = lambda fname: fname.startswith("tests")
+        _filter_pkg = lambda fname: _is_in_test(fname) or (
+            fname.startswith("src/torchmetrics") and "__init__.py" not in fname
+        )
         files_pkg = [fn for fn in files if _filter_pkg(fn)]
         if not files_pkg:
             return _return_all
 
         # parse domains
-        def _crop_path(fname: str, paths: list[str]) -> str:
+        def _crop_path(fname: str, paths: tuple[str] = ("src/torchmetrics/", "tests/unittests/", "functional/")) -> str:
             for p in paths:
                 fname = fname.replace(p, "")
             return fname
 
-        files_pkg = [_crop_path(fn, ["src/torchmetrics/", "tests/unittests/", "functional/"]) for fn in files_pkg]
+        files_pkg = [_crop_path(fn) for fn in files_pkg]
         # filter domain names
         tm_modules = [fn.split("/")[0] for fn in files_pkg if "/" in fn]
         # filter general (used everywhere) sub-packages
@@ -179,6 +194,23 @@ def _crop_path(fname: str, paths: list[str]) -> str:
             raise ValueError(f"Missing following paths: {not_exists}")
         return " ".join(test_modules)
 
+    @staticmethod
+    def move_new_packages(dir_cache: str, dir_local: str, dir_staging: str) -> None:
+        """Move unique packages from local folder to staging."""
+        assert os.path.isdir(dir_cache), f"Missing folder with saved packages: '{dir_cache}'"  # noqa: S101
+        assert os.path.isdir(dir_local), f"Missing folder with local packages: '{dir_local}'"  # noqa: S101
+        assert os.path.isdir(dir_staging), f"Missing folder for staging: '{dir_staging}'"  # noqa: S101
+
+        import shutil
+
+        for pkg in os.listdir(dir_local):
+            if not os.path.isfile(pkg):
+                continue
+            if pkg in os.listdir(dir_cache):
+                continue
+            logging.info(f"Moving '{pkg}' to staging...")
+            shutil.move(os.path.join(dir_cache, pkg), os.path.join(dir_staging, pkg))
+
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)

.github/workflows/_merge_cache.yml

@@ -0,0 +1,57 @@
+name: Collect new packages and upload cache
+
+on:
+  workflow_call:
+    inputs:
+      pypi-key:
+        description: cache restore/dump key
+        required: false
+        type: string
+        default: "pypi-packages"
+      pypi-dir:
+        description: location of local PyPI cache
+        required: false
+        type: string
+        default: "_ci-cache_PyPI"
+      cache-artifact-appendix:
+        description: "unique name for the job"
+        required: true
+        type: string
+
+jobs:
+  merge-caches:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download 📥 artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: ${{ inputs.pypi-key }}-run-${{ inputs.cache-artifact-appendix }}*
+          merge-multiple: true
+          path: _local-packages
+      - name: Cache pull packages
+        uses: actions/cache/restore@v3
+        with:
+          enableCrossOsArchive: true
+          path: ${{ inputs.pypi-dir }}
+          key: ${{ inputs.pypi-key }}
+
+      - name: show 📦
+        run: |
+          # create the directory if it doesn't exist - no artifact were found
+          mkdir -p _local-packages
+          ls -lh _local-packages
+          ls -lh ${{ inputs.pypi-dir }}
+          # count files in the staging dir
+          file_count=$(ls -1 "_local-packages/" | wc -l)
+          echo "NUM_PACKAGES=$file_count" >> $GITHUB_ENV
+      - name: Move collected 📦
+        if: env.NUM_PACKAGES != 0
+        run: mv _local-packages/* ${{ inputs.pypi-dir }}
+
+      - name: Cache push packages
+        if: env.NUM_PACKAGES != 0
+        uses: actions/cache/save@v3
+        with:
+          enableCrossOsArchive: true
+          path: ${{ inputs.pypi-dir }}
+          key: ${{ inputs.pypi-key }}

.github/workflows/ci-integrate.yml

@@ -36,7 +36,7 @@ jobs:
           - { python-version: "3.10", requires: "latest", os: "ubuntu-22.04" }
           # - { python-version: "3.10", requires: "latest", os: "macOS-14" } # M1 machine # todo: crashing for MPS out of memory
     env:
-      PYTORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+      PYTORCH_URL: "http://download.pytorch.org/whl/cpu/"
       FREEZE_REQUIREMENTS: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
       PYPI_CACHE: "_ci-cache_PyPI"
 
@@ -64,7 +64,7 @@ jobs:
           # this was updated in `source cashing` by optional oldest
           cat requirements/_integrate.txt
           # to have install pyTorch
-          pip install -e . "setuptools==69.5.1" --find-links=${PYTORCH_URL}
+          pip install -e . "setuptools==69.5.1" --extra-index-url=${PYTORCH_URL}
 
           # adjust version to PT ecosystem based on installed TM
           python -m wget https://raw.githubusercontent.com/Lightning-AI/utilities/main/scripts/adjust-torch-versions.py
@@ -73,7 +73,7 @@ jobs:
 
           # install package and dependencies
           pip install -e . -r requirements/_tests.txt -r requirements/_integrate.txt \
-            --find-links=${PYTORCH_URL} --find-links=${PYPI_CACHE} \
+            --extra-index-url="${PYTORCH_URL}" --find-links="${PYPI_CACHE}" \
             --upgrade-strategy eager
           pip list