diff --git a/.github/workflows/self-comment-slow-ci.yml b/.github/workflows/self-comment-slow-ci.yml
index b7ea7f862eec19..e3a974075ca52c 100644
--- a/.github/workflows/self-comment-slow-ci.yml
+++ b/.github/workflows/self-comment-slow-ci.yml
@@ -4,40 +4,17 @@ on:
   issue_comment:
     types: 
       - created
-      - labeled
     branches-ignore:
       - main
   pull_request_review_comment:
     types: [created, deleted]
-  pull_request:
+  # pull_request:
 concurrency:
   # But this will cancel the current job no ? Because they have the same PR number ... (any comment even not the target one)
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}-${{ contains(github.event.comment.body, 'run-slow') }}
   cancel-in-progress: ${{ (github.event.issue.pull_request != null || github.event.pull_request != null) }}
 
 jobs:
-  reply-and-trigger:
-    runs-on: ubuntu-latest
-    permissions: write-all
-    steps:
-      - name: Trigger specific job
-        run: |
-          echo "${{ github.event.comment.body }}"
-          echo "${{ startsWith(github.event.comment.body, 'run-slow') }}"
-          echo "hello 4"
-          echo "${{ github.head_ref }}"
-          echo "${{ github.ref }}"
-          echo "${{ github.sha }}"
-          echo "${{ github.event.pull_request.number }}"
-          echo "${{ github.event.issue.number }}"
-          echo "${{ github.event.issue.pull_request }}"
-          echo "${{ github.event.pull_request }}"
-          echo "${{ github.event.issue.pull_request != null }}"
-          echo "${{ github.event.pull_request != null }}"
-          echo "${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}-${{ contains(github.event.comment.body, 'run-slow') }}"
-          echo "${{ github.event.pull_request.head.sha }}"
-          echo "${{ github.event.pull_request.merge_commit_sha }}"
-
   get-pr-number:
     runs-on: ubuntu-22.04
     name: Get PR number
@@ -67,34 +44,6 @@ jobs:
         id: set_pr_number
         run: echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> "$GITHUB_OUTPUT"
 
-  check-pr-number:
-    runs-on: ubuntu-latest
-    needs: get-pr-number
-    steps:
-      - env:
-          PR_NUMBER: ${{needs.get-pr-number.outputs.PR_NUMBER}}
-        run: |
-          echo "$PR_NUMBER"
-          echo "${{ env.PR_NUMBER }}"
-
-  check-pr-merge-branch:
-    runs-on: ubuntu-latest
-    needs: get-pr-number
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: "0"
-          ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"
-
-      - name: Checkout PR merge branch
-        env:
-          PR_NUMBER: ${{needs.get-pr-number.outputs.PR_NUMBER}}
-        run: |
-            git fetch
-            git fetch origin refs/pull/$PR_NUMBER/merge:refs/remotes/pull/$PR_NUMBER/merge
-            git checkout refs/remotes/pull/$PR_NUMBER/merge
-            git log -n 5
-
   # use a python script to handle this complex logic
   # case 1: `run-slow` (auto. infer with limited number of models, but in particular, new model)
   # case 2: `run-slow model_1, model_2`
@@ -103,6 +52,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: get-pr-number
     if: ${{ needs.get-pr-number.outputs.PR_NUMBER != ''}}
+    permissions: write-all
     outputs:
       models: ${{ steps.models_to_run.outputs.models }}
     steps:
@@ -111,6 +61,17 @@ jobs:
           fetch-depth: "0"
           ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"
 
+      - name: Reply to the comment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            repos/${{ github.repository }}/issues/${{ needs.get-pr-number.outputs.PR_NUMBER }}/comments \
+            -f "body=This comment contains run-slow, running the specified job..."
+
       - name: Get models to test
         run: |
           python -m pip install GitPython
@@ -123,27 +84,55 @@ jobs:
           echo "${{ env.models }}"
           echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
 
-  show-tests:
-    runs-on: ubuntu-latest
-    name: Show model to test
+  create_run:
+    name: Create run
+    if: ${{ needs.get-tests.outputs.models != '[]' }}
     needs: get-tests
+    permissions: write-all
+    runs-on: ubuntu-latest
     steps:
-      - name: Show models to test
+      - name: Create Run
+        id: create_run
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
-          echo "${{ needs.get-tests.outputs.models }}"
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+            -f "target_url=$GITHUB_RUN_URL" -f "state=pending" -f "description=Slow CI job" -f "context=pytest/custom-tests"
+
+  run_models_gpu_dummy:
+      name: Run all tests for the model
+      # Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
+      # (either a new model PR or via a commit message)
+      if: ${{ needs.get-tests.outputs.models != '[]' }}
+      needs: [get-tests, create_run]
+      strategy:
+        fail-fast: false
+        matrix:
+          folders: ${{ fromJson(needs.get-tests.outputs.models) }}
+          machine_type: [aws-g4dn-2xlarge-cache, aws-g4dn-12xlarge-cache]
+      runs-on: ubuntu-latest
+      steps:
+      - name: Echo input and matrix info
+        shell: bash
+        run: |
+          echo "${{ matrix.folders }}"
 
   run_models_gpu:
       name: Run all tests for the model
       # Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
       # (either a new model PR or via a commit message)
       if: ${{ needs.get-tests.outputs.models != '[]' }}
-      needs: get-tests
+      needs: [get-tests, create_run]
       strategy:
         fail-fast: false
         matrix:
           folders: ${{ fromJson(needs.get-tests.outputs.models) }}
           machine_type: [aws-g4dn-2xlarge-cache, aws-g4dn-12xlarge-cache]
-#      runs-on: ubuntu-latest
       runs-on:
          group: '${{ matrix.machine_type }}'
       container:
@@ -155,80 +144,107 @@ jobs:
         run: |
           echo "${{ matrix.folders }}"
 
-#  git-checkout:
-#      runs-on: ubuntu-22.04
-#      name: Check PR merge branch
-#      steps:
-#        - name: Get PR number
-#          run: |
-#            if
-#
-#        - uses: actions/checkout@v4
-#          with:
-#            fetch-depth: "0"
-#            # ref: ${{ github.event.pull_request.head.sha }}
-#            ref: "refs/pull/${{ github.event.pull_request.number }}/merge"
-#
-#        - name: Checkout
-#          run: |
-#            git fetch
-#            git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge
-#            git checkout pull/${{ github.event.pull_request.number }}/merge
-#            git log -n 5
-#
-#        - name: Checkout 2
-#          run: |
-#            git fetch
-#            git fetch origin refs/pull/${{ github.event.pull_request.number }}/merge:refs/remotes/pull/${{ github.event.pull_request.number }}/merge
-#            git checkout refs/remotes/pull/${{ github.event.pull_request.number }}/merge
-#            git log -n 5
-#
-#        - name: Checkout 3
-#          run: |
-#            git fetch
-#            git fetch origin refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/pull/${{ github.event.pull_request.number }}/head
-#            git checkout refs/remotes/pull/${{ github.event.pull_request.number }}/head
-#            git log -n 6
-
-#      - name: Set up GitHub CLI
-#        uses: actions/checkout@v4
-#
-#      - name: Reply to the comment
-#        env:
-#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#        run: |
-#          gh api \
-#            --method POST \
-#            -H "Accept: application/vnd.github+json" \
-#            -H "X-GitHub-Api-Version: 2022-11-28" \
-#            /repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
-#            -f "body=This comment contains /pytest, running the specified job..."
-#
-#      - name: Create  Run
-#        id: create_run
-#        env:
-#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#          GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-#        run: |
-#          gh api \
-#            --method POST \
-#            -H "Accept: application/vnd.github+json" \
-#            -H "X-GitHub-Api-Version: 2022-11-28" \
-#            /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
-#            -f "target_url=$GITHUB_RUN_URL" -f "state=pending" -f "description=Custom CI job" -f "context=pytest/custom-tests"
-#      - name: Trigger specific job
-#        run: |
-#          echo "Running the specific job because /pytest was found"
-#          # Add the logic to trigger your specific job here
-
-#      - name: Update Check Run Status
-#        env:
-#          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-#          GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
-#        run: |
-#          gh api \
-#            --method POST \
-#            -H "Accept: application/vnd.github+json" \
-#            -H "X-GitHub-Api-Version: 2022-11-28" \
-#            /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
-#            -f "target_url=$GITHUB_RUN_URL" -f "state=success" -f "description=Custom CI job" -f "context=pytest/custom-tests"
+      - name: Echo folder ${{ matrix.folders }}
+        shell: bash
+        # For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
+        # set the artifact folder names (because the character `/` is not allowed).
+        run: |
+          echo "${{ matrix.folders }}"
+          matrix_folders=${{ matrix.folders }}
+          matrix_folders=${matrix_folders/'models/'/'models_'}
+          echo "$matrix_folders"
+          echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
+
+      - name: Update clone
+        working-directory: /transformers
+        run: git fetch && git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge && git checkout pull/${{ github.event.pull_request.number }}/merge
+
+      - name: Reinstall transformers in edit mode (remove the one installed during docker image build)
+        working-directory: /transformers
+        run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
+
+      - name: NVIDIA-SMI
+        run: |
+          nvidia-smi
+
+      - name: Set `machine_type` for report and artifact names
+        working-directory: /transformers
+        shell: bash
+        run: |
+          echo "${{ matrix.machine_type }}"
+          if [ "${{ matrix.machine_type }}" = "aws-g4dn-2xlarge-cache" ]; then
+            machine_type=single-gpu
+          elif [ "${{ matrix.machine_type }}" = "aws-g4dn-12xlarge-cache" ]; then
+            machine_type=multi-gpu
+          else
+            machine_type=${{ matrix.machine_type }}
+          fi
+          echo "$machine_type"
+          echo "machine_type=$machine_type" >> $GITHUB_ENV    
+
+      - name: Environment
+        working-directory: /transformers
+        run: |
+          python3 utils/print_env.py
+
+      - name: Show installed libraries and their versions
+        working-directory: /transformers
+        run: pip freeze
+
+      - name: Run all tests on GPU
+        working-directory: /transformers
+        run: |
+          export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
+          echo $CUDA_VISIBLE_DEVICES
+          python3 -m pytest -v -rsfE --make-reports=${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
+
+      - name: Failure short reports
+        if: ${{ failure() }}
+        continue-on-error: true
+        run: cat /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
+
+      - name: Make sure report directory exists
+        shell: bash
+        run: |
+          mkdir -p /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
+          echo "hello" > /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
+          echo "${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
+
+      - name: "Test suite reports artifacts: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
+          path: /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
+
+  update_run_status:
+    name: Update Check Run Status
+    needs: [run_models_gpu]
+    permissions: write-all
+    if: always()
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - name: Get `run_models_gpu` job status
+        run: |
+          echo "${{ needs.run_models_gpu.result }}"
+          if [ "${{ needs.run_models_gpu.result }}" = "cancelled" ]; then
+            echo "STATUS=failure" >> $GITHUB_ENV
+          elif [ "${{ needs.run_models_gpu.result }}" = "skipped" ]; then
+            echo "STATUS=success" >> $GITHUB_ENV
+          else
+            echo "STATUS=${{ needs.run_models_gpu.result }}" >> $GITHUB_ENV
+          fi
+
+      - name: Update run's status
+        run: |
+          echo "${{ needs.run_models_gpu.result }}"
+          echo "${{ env.STATUS }}"
+          gh api \
+            --method POST \
+            -H "Accept: application/vnd.github+json" \
+            -H "X-GitHub-Api-Version: 2022-11-28" \
+            repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+            -f "target_url=$GITHUB_RUN_URL" -f "state=${{ env.STATUS }}" -f "description=Slow CI job" -f "context=pytest/custom-tests"