From aea57a2d1dbef02fa13d2ab2fea3e9f9af3001dd Mon Sep 17 00:00:00 2001
From: sule <sule.sl@antgroup.com>
Date: Wed, 5 Feb 2025 19:26:00 +0800
Subject: [PATCH 1/3] [CI] Add Ray ci

---
 .github/workflows/ray-ci.yml        | 550 ++++++++++++++++++++++++
 build-docker-ant.sh                 |   2 +-
 ci/docker/core.build.ant.Dockerfile |  16 +
 ci/ray_ci/builder.py                |   1 -
 ci/ray_ci/linux_container.py        |  12 +-
 ci/ray_ci/report_gen.py             | 625 ++++++++++++++++++++++++++++
 ci/ray_ci/tester.py                 |  31 +-
 ci/ray_ci/tester_container.py       |   2 -
 ci/ray_ci/tests.env.Dockerfile      |   2 +-
 ci/run/bazel_export_options         |   8 +-
 10 files changed, 1207 insertions(+), 42 deletions(-)
 create mode 100644 .github/workflows/ray-ci.yml
 create mode 100644 ci/docker/core.build.ant.Dockerfile
 create mode 100644 ci/ray_ci/report_gen.py

diff --git a/.github/workflows/ray-ci.yml b/.github/workflows/ray-ci.yml
new file mode 100644
index 0000000000000..6577348007edf
--- /dev/null
+++ b/.github/workflows/ray-ci.yml
@@ -0,0 +1,550 @@
+name: Ray CI
+
+on:
+  pull_request:
+    branches:
+      - main
+
+
+env:
+  PYTHON_VERSION: '3.11'
+  DOCKER_BUILDKIT: 1
+  RAY_CI_POST_WHEEL_TESTS: 1
+
+
+jobs:
+  start-ecs-runner:
+    name: Start ECS Runner
+    runs-on: ubuntu-latest
+    environment: aliyun
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: false
+    outputs:
+      leased_instance_id: ${{ steps.lease_instance.outputs.instance_id }}
+      leased_disk_id: ${{ steps.lease_instance.outputs.disk_id }}
+    steps:
+      - name: Configure Aliyun CLI
+        run: |
+          wget https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-amd64.tgz
+          tar -xvf aliyun-cli-linux-latest-amd64.tgz
+          ./aliyun configure set --profile default \
+            --access-key-id ${{ secrets.ALIYUN_ACCESS_KEY }} \
+            --access-key-secret ${{ secrets.ALIYUN_ACCESS_SECRET }} \
+            --region us-west-1
+
+      - name: Lease and Start Instance
+        id: lease_instance
+        run: |
+          # Parse instance:disk pairs
+          IFS=',' read -ra INSTANCE_PAIRS <<< "${{ secrets.ALIYUN_ECS_INSTANCE_IDS }}"
+          
+          while true; do
+            for pair in "${INSTANCE_PAIRS[@]}"; do
+              IFS=':' read -r instance_id disk_id <<< "$pair"
+              
+              # Check instance status
+              status=$(./aliyun ecs DescribeInstances --InstanceIds '["'"$instance_id"'"]' --region us-west-1 | jq -r '.Instances.Instance[0].Status')
+              
+              if [ "$status" = "Stopped" ]; then
+                # Try to start this instance
+                max_attempts=30
+                for i in $(seq 1 $max_attempts); do
+                  if ./aliyun ecs StartInstance --InstanceId "$instance_id" --region us-west-1; then
+                    echo "StartInstance succeeded on attempt $i"
+                    # Wait for instance to be running
+                    while true; do
+                      current_status=$(./aliyun ecs DescribeInstances --InstanceIds '["'"$instance_id"'"]' --region us-west-1 | jq -r '.Instances.Instance[0].Status')
+                      if [ "$current_status" = "Running" ]; then
+                        break
+                      fi
+                      sleep 10
+                    done
+                    
+                    # Try to attach disk
+                    for j in $(seq 1 $max_attempts); do
+                      if ./aliyun ecs AttachDisk --InstanceId "$instance_id" --DiskId "$disk_id" --region us-west-1; then
+                        echo "AttachDisk succeeded on attempt $j"
+                        # Set outputs for use in subsequent jobs
+                        echo "instance_id=$instance_id" >> $GITHUB_OUTPUT
+                        echo "disk_id=$disk_id" >> $GITHUB_OUTPUT
+                        exit 0
+                      else
+                        echo "AttachDisk failed on attempt $j"
+                        if [ $j -eq $max_attempts ]; then
+                          echo "All AttachDisk attempts failed - trying next instance"
+                          break
+                        fi
+                        sleep 10
+                      fi
+                    done
+                    break
+                  else
+                    echo "StartInstance failed on attempt $i"
+                    if [ $i -eq $max_attempts ]; then
+                      echo "All StartInstance attempts failed - trying next instance"
+                    fi
+                    sleep 10
+                  fi
+                done
+              fi
+            done
+            
+            echo "No available instances found in this iteration, waiting before retrying..."
+            sleep 60  # Wait 1 minute before trying again
+          done
+
+  check-permissions:
+    name: Check User Permissions
+    needs: start-ecs-runner
+    runs-on: ${{ needs.start-ecs-runner.outputs.leased_instance_id }}
+    outputs:
+      allowed: ${{ steps.check_user.outputs.allowed }}
+    steps:
+      - name: Check if user is allowed
+        id: check_user
+        run: |
+          if [ -f "~/.workflow.cfg" ]; then
+            if grep -qE "^allowed_users=(${GITHUB_ACTOR}|.*,${GITHUB_ACTOR}|${GITHUB_ACTOR},.*|.*,${GITHUB_ACTOR},.*)\$" ~/.workflow.cfg; then
+              echo "allowed=true" >> $GITHUB_OUTPUT
+            else
+              echo "allowed=false" >> $GITHUB_OUTPUT
+              echo "User ${GITHUB_ACTOR} is not authorized to trigger this workflow"
+            fi
+          else
+            echo "allowed=false" >> $GITHUB_OUTPUT
+          fi
+
+  build-base-images:
+    name: Build Base Images
+    needs: [check-permissions, start-ecs-runner]
+    if: needs.check-permissions.outputs.allowed == 'true'
+    runs-on: ${{ needs.start-ecs-runner.outputs.leased_instance_id }}
+    strategy:
+      matrix:
+        python: ['3.11']
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+    
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Check Test Image Existence
+      id: check_test_image
+      run: |
+        if docker manifest inspect ghcr.io/${{ github.repository }}/oss-ci-base_test-py${{ matrix.python }} > /dev/null 2>&1; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+        fi
+
+
+    - name: Build Base Test Image
+      if: steps.check_test_image.outputs.exists != 'true'
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ci/docker/base.test.Dockerfile
+        tags: |
+          ghcr.io/${{ github.repository }}/oss-ci-base_test-py${{ matrix.python }}
+        push: true
+
+    - name: Check Build Image Existence
+      id: check_build_image
+      run: |
+        if docker manifest inspect ghcr.io/${{ github.repository }}/oss-ci-base_build-py${{ matrix.python }} > /dev/null 2>&1; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+        fi
+    - name: Build OSS CI Base
+      if: steps.check_build_image.outputs.exists != 'true'
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ci/docker/base.build.Dockerfile
+        tags: ghcr.io/${{ github.repository }}/oss-ci-base_build-py${{ matrix.python }}
+        build-args: |
+          DOCKER_IMAGE_BASE_TEST=ghcr.io/${{ github.repository }}/oss-ci-base_test-py${{ matrix.python }}
+        push: true
+
+
+  core-tests:
+    name: Core Tests
+    env:
+      BAZEL_DIR: ${{ github.workspace }}/../tmp/bazel/${{ github.sha }}
+      RAYCI_BUILD_ID: main
+      RAY_DIR: ${{ github.workspace }}
+      RAYCI_CHECKOUT_DIR: ${{ github.workspace }}
+      COMMIT_HASH: ${{ github.sha }}
+      REPORT_LABEL_1: core_python_tests_1
+      REPORT_LABEL_2: core_python_tests_2
+      REPORT_LABEL_3: core_python_tests_3
+      REPORT_LABEL_4: core_cpp_tests
+      REPORT_LABEL_5: dashboard_tests
+      REPORT_LABEL_6: workflow_tests
+      REPORT_LABEL_7: debug_tests
+      REPORT_LABEL_8: asan_tests
+    needs: [build-base-images, check-permissions, start-ecs-runner]
+    if: needs.check-permissions.outputs.allowed == 'true'
+    runs-on: ${{ needs.start-ecs-runner.outputs.leased_instance_id }}
+    strategy:
+      matrix:
+        python: ['3.11']
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+    
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Check Core Image Existence
+      id: check_core_image
+      run: |
+        if docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild"; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+        fi
+    - name: Build core CI Base
+      if: steps.check_core_image.outputs.exists != 'true'
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ci/docker/core.build.Dockerfile
+        tags: |
+          localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild
+        build-args: |
+          RAYCI_IS_GPU_BUILD=false
+          BUILDKITE=true
+          DOCKER_IMAGE_BASE_BUILD=ghcr.io/${{ github.repository }}/oss-ci-base_build-py${{ matrix.python }}
+        push: true
+
+    - name: Start local registry
+      if: '!cancelled()'
+      run: |
+        # Check if registry container exists and handle accordingly
+        if ! docker ps -a --format '{{.Names}}' | grep -q '^local-registry$'; then
+          docker run -d -p 5000:5000 --name local-registry registry:2
+        elif ! docker ps --format '{{.Names}}' | grep -q '^local-registry$'; then
+          docker start local-registry
+        fi
+    - name: Push core build image to local registry
+      if: '!cancelled()'
+      run: |
+        max_attempts=3
+        for i in $(seq 1 $max_attempts); do
+          if docker push localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild; then
+            echo "Docker push succeeded on attempt $i"
+            break
+          else
+            echo "Docker push failed on attempt $i"
+            if [ $i -eq $max_attempts ]; then
+              echo "All docker push attempts failed - exiting"
+              exit 1
+            fi
+            sleep 10
+          fi
+        done
+    
+    - name: Install bazel
+      if: '!cancelled()'
+      run: |
+        bash ci/env/install-bazel.sh
+    - name: Check Manylinux Image Existence
+      id: check_manylinux_image
+      run: |
+        if docker image ls --format '{{.Repository}}:{{.Tag}}' | grep -q "localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-manylinux"; then
+          echo "exists=true" >> $GITHUB_OUTPUT
+        else
+          echo "exists=false" >> $GITHUB_OUTPUT
+        fi
+    - name: Build Manylinux Image
+      if: steps.check_manylinux_image.outputs.exists != 'true'
+      uses: docker/build-push-action@v3
+      with:
+        context: .
+        file: ci/docker/manylinux.Dockerfile
+        tags: |
+          localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-manylinux
+        push: true
+        build-args: |
+          BUILDKITE_BAZEL_CACHE_URL
+          HOSTTYPE=x86_64
+    - name: Run Core Python Tests (1)
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+      run: |
+        echo "Running core: python tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core \
+          --workers 1 --worker-id 0 --parallelism-per-worker 12 \
+          --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu,large_size_python_tests_shard_2 \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }}) \
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_1 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run Core Python Tests (2)
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running core: python tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/dag/... python/ray/autoscaler/v2/... core \
+          --workers 1 --worker-id 0 --parallelism-per-worker 12 --skip-ray-installation \
+          --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu,large_size_python_tests_shard_2 \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }}) \
+          --skip-ray-installation
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_2 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run Core Python Tests (3)
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        #echo "Running core: python tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core \
+          --workers 1 --worker-id 0 --parallelism-per-worker 12 --skip-ray-installation \
+          --only-tags use_all_core \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_3 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Update Test Image
+      if: '!cancelled()'
+      run: |
+        docker build -t localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild -f ci/docker/core.build.ant.Dockerfile \
+          --build-arg DOCKER_IMAGE_BASE=localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild \
+          .
+    - name: Run Core Cpp Tests
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running core: cpp tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type clang \
+          --workers 1 --worker-id 0 --parallelism-per-worker 12 --skip-ray-installation \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_4 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run Dashboard Tests
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running dashboard tests..."
+        bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... core \
+          --parallelism-per-worker 12 --skip-ray-installation \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_5 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run Workflow Tests
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running workflow tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/workflow/... core \
+          --workers 1 --worker-id 0 \
+          --except-tags use_all_core \
+          --parallelism-per-worker 12 \
+          --skip-ray-installation \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/workflow/... core \
+          --workers 1 --worker-id 0 \
+          --skip-ray-installation \
+          --only-tags use_all_core \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+        
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_6 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run Debug Tests
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running debug tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core \
+          --build-type debug \
+          --parallelism-per-worker 12 \
+          --only-tags debug_tests \
+          --except-tags kubernetes,manual \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }}) \
+          --skip-ray-installation
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_7 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Run ASAN Tests
+      if: '!cancelled()'
+      env:
+        RAYCI_WORK_REPO: localhost:5000/citemp
+
+      run: |
+        echo "Running ASAN tests..."
+        bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core \
+          --build-type asan \
+          --parallelism-per-worker 12 \
+          --only-tags asan_tests \
+          --except-tags kubernetes,manual \
+          --skip-ray-installation \
+          --test-env=BAZEL_DIR=$(realpath ${{ env.BAZEL_DIR }})
+    - name: Grant Permissions
+      if: '!cancelled()'
+      run: |
+        sudo chmod -R 777 ${{ env.BAZEL_DIR }}
+    - name: Generate Report For Label
+      if: '!cancelled()'
+      env:
+        REPORT_LABEL: ${{ env.REPORT_LABEL_8 }}
+      run: |
+        python ci/ray_ci/report_gen.py
+    - name: Workaround
+      if: '!cancelled()'
+      run: |
+        echo "artifacts_path=$(realpath ${{ env.BAZEL_DIR }})" >> $GITHUB_ENV
+ 
+    - name: Generate Report
+      if: '!cancelled()'
+      run: |
+        python ci/ray_ci/report_gen.py summary ${{ env.REPORT_LABEL_1 }} ${{ env.REPORT_LABEL_2 }} ${{ env.REPORT_LABEL_3 }} ${{ env.REPORT_LABEL_4 }} ${{ env.REPORT_LABEL_5 }} ${{ env.REPORT_LABEL_6 }} ${{ env.REPORT_LABEL_7 }} ${{ env.REPORT_LABEL_8 }}
+    - name: Cleanup Containers
+      if: always()
+      continue-on-error: true
+      run: |
+        docker ps -a --filter ancestor=localhost:5000/citemp:${{ env.RAYCI_BUILD_ID }}-corebuild -q | xargs -r docker rm --force || true
+        # Clean up any dangling containers
+        docker ps -a --filter status=exited -q | xargs -r docker rm || true
+    - name: Upload Test Reports
+      id: artifact-upload-step
+      continue-on-error: true
+      if: '!cancelled()'
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-reports-${{ github.sha }}
+        path: ${{ env.artifacts_path }}/reports/
+
+    - name: Create Summary
+      if: '!cancelled()'
+      run: |
+        echo "### 📊 Test Reports Available" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "View detailed test results at:" >> $GITHUB_STEP_SUMMARY
+        echo "🔗 [Ant Ray Dashboard](https://ant-ray.streamlit.app?run_id=${{ github.run_id }}&artifact_id=${{ steps.artifact-upload-step.outputs.artifact-id }})" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "This link contains test reports and analysis for commit \`${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY
+  stop-ecs-runner:
+    name: Stop ECS Instance
+    environment: aliyun
+    needs:
+      - start-ecs-runner
+      - check-permissions
+      - build-base-images
+      - core-tests
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Configure Aliyun CLI
+        run: |
+          wget https://aliyuncli.alicdn.com/aliyun-cli-linux-latest-amd64.tgz
+          tar -xvf aliyun-cli-linux-latest-amd64.tgz
+          ./aliyun configure set --profile default \
+            --access-key-id ${{ secrets.ALIYUN_ACCESS_KEY }} \
+            --access-key-secret ${{ secrets.ALIYUN_ACCESS_SECRET }} \
+            --region us-west-1
+          # Add retry logic for StopInstance
+          max_attempts=3
+          for i in $(seq 1 $max_attempts); do
+            if ./aliyun ecs StopInstance --InstanceId ${{ needs.start-ecs-runner.outputs.leased_instance_id }} --ForceStop true --region us-west-1; then
+              echo "StopInstance succeeded on attempt $i"
+              break
+            else
+              echo "StopInstance failed on attempt $i"
+              if [ $i -eq $max_attempts ]; then
+                echo "All attempts failed - exiting"
+                exit 1
+              fi
+              sleep 10
+            fi
+          done
+          while true; do
+            current_status=$(./aliyun ecs DescribeInstances --InstanceIds '["'"${{ needs.start-ecs-runner.outputs.leased_instance_id }}"'"]' --region us-west-1 | jq -r '.Instances.Instance[0].Status')
+            if [ "$current_status" = "Stopped" ]; then
+              break
+            fi
+            sleep 10
+          done
\ No newline at end of file
diff --git a/build-docker-ant.sh b/build-docker-ant.sh
index 0cbe372d785cd..b3c8dc4ad83db 100644
--- a/build-docker-ant.sh
+++ b/build-docker-ant.sh
@@ -94,7 +94,7 @@ docker run --rm \
     antgroup/base-deps:dev$GPU \
     pip download --no-cache-dir --quiet --no-deps --only-binary=:all: "$ANT_RAY_PACKAGE" -d /wheels
 
-WHEEL_FILE=$(ls "$RAY_BUILD_DIR"/.whl/*.whl 2>/dev/null | head -n1)
+WHEEL_FILE=$(find "$RAY_BUILD_DIR/.whl" -name '*.whl' -type f -print -quit 2>/dev/null)
 if [[ -z "$WHEEL_FILE" ]]; then
     echo "Error: No wheel downloaded for ant-ray" >&2
     exit 1
diff --git a/ci/docker/core.build.ant.Dockerfile b/ci/docker/core.build.ant.Dockerfile
new file mode 100644
index 0000000000000..03a93f3a23365
--- /dev/null
+++ b/ci/docker/core.build.ant.Dockerfile
@@ -0,0 +1,16 @@
+ARG DOCKER_IMAGE_BASE
+FROM $DOCKER_IMAGE_BASE
+
+WORKDIR /rayci
+
+RUN <<EOF
+#!/bin/bash
+
+set -euo pipefail
+
+chmod 755 /usr/lib/sudo/sudoers.so
+apt-get update && apt-get install -y --reinstall sudo
+
+BUILD=1 DL=1 ./ci/env/install-dependencies.sh
+
+EOF
diff --git a/ci/ray_ci/builder.py b/ci/ray_ci/builder.py
index 5ac8960fa2b4d..8dac360fc12fc 100644
--- a/ci/ray_ci/builder.py
+++ b/ci/ray_ci/builder.py
@@ -84,7 +84,6 @@ def main(
     """
     Build a wheel or jar artifact
     """
-    docker_login(_DOCKER_ECR_REPO.split("/")[0])
     ci_init()
     if artifact_type == "wheel":
         logger.info(f"Building wheel for {python_version}")
diff --git a/ci/ray_ci/linux_container.py b/ci/ray_ci/linux_container.py
index 91c5dbfa2b9da..4cdb4bbccf887 100644
--- a/ci/ray_ci/linux_container.py
+++ b/ci/ray_ci/linux_container.py
@@ -47,8 +47,8 @@ def install_ray(self, build_type: Optional[str] = None) -> List[str]:
                 "-t",
                 self._get_docker_image(),
                 "-f",
-                "/ray/ci/ray_ci/tests.env.Dockerfile",
-                "/ray",
+                f"{os.environ.get('RAY_DIR')}/ci/ray_ci/tests.env.Dockerfile",
+                os.environ.get("RAY_DIR"),
             ],
             env=env,
             stdout=sys.stdout,
@@ -84,4 +84,10 @@ def get_run_command_extra_args(
         return extra_args
 
     def get_artifact_mount(self) -> Tuple[str, str]:
-        return ("/tmp/artifacts", "/artifact-mount")
+        os.makedirs(
+            os.path.join(os.environ.get("BAZEL_DIR"), "artifacts"), exist_ok=True
+        )
+        return (
+            os.path.join(os.environ.get("BAZEL_DIR"), "artifacts"),
+            os.environ.get("BAZEL_DIR"),
+        )
diff --git a/ci/ray_ci/report_gen.py b/ci/ray_ci/report_gen.py
new file mode 100644
index 0000000000000..8d8fa55e4f459
--- /dev/null
+++ b/ci/ray_ci/report_gen.py
@@ -0,0 +1,625 @@
+import os
+import shutil
+from pathlib import Path
+from textwrap import dedent
+import re
+import json
+
+
+BASE_DIR = os.getenv("BAZEL_DIR")
+COMMIT_HASH = os.getenv("COMMIT_HASH")
+REPORT_DIR = Path(f"{BASE_DIR}/reports/")
+LOG_SOURCE_DIR = Path(f"{BASE_DIR}/artifacts/bazel_event_logs/")
+FAILED_LOGS_DIR = Path(f"{BASE_DIR}/artifacts/failed_test_logs/")
+REPORT_NAME = os.getenv("REPORT_LABEL")
+
+MODERN_CSS = """
+<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+<style>
+    :root {
+        --bs-body-bg: #0f172a;
+        --bs-body-color: #e2e8f0;
+        --bs-primary: #3b82f6;
+        --bs-border-color: #334155;
+    }
+    body { 
+        background-color: var(--bs-body-bg);
+        color: var(--bs-body-color);
+    }
+    .card {
+        background-color: #1e293b;
+        border: 1px solid var(--bs-border-color);
+        box-shadow: 0 1px 3px rgba(0,0,0,0.2);
+    }
+    .list-group-item {
+        background-color: #1e293b;
+        color: #e2e8f0;
+        border-color: var(--bs-border-color);
+    }
+    /* Fixed navigation styling */
+    .nav-sidebar {
+        width: 280px;
+        position: fixed;
+        left: 0;
+        top: 0;
+        bottom: 0;
+        overflow-y: auto;
+        padding: 1rem;
+        background-color: #1e293b;
+        border-right: 1px solid var(--bs-border-color);
+        padding-right: 0.5rem;
+        z-index: 1000;
+    }
+    .main-content {
+        margin-left: 280px;
+        padding: 2rem;
+        min-height: 100vh;
+        overflow-y: auto;
+        height: 100vh;
+    }
+    /* Search bar styling */
+    .search-container {
+        position: sticky;
+        top: 0;
+        z-index: 1020;
+        background-color: var(--bs-body-bg);
+        padding: 1rem 0;
+        margin: -1rem -0.5rem 1rem;
+    }
+    
+    /* Search input styling */
+    .search-container input {
+        background-color: #1e293b;
+        border-color: var(--bs-border-color);
+        color: white;
+    }
+    
+    .search-container input::placeholder {
+        color: rgba(255, 255, 255, 0.6);
+    }
+    /* Active state styling */
+    .list-group-item.active {
+        background-color: #1e40af !important;
+        border-color: #3b82f6 !important;
+        color: white !important;
+        border-left: 4px solid var(--bs-primary);
+    }
+    /* Improved code blocks */
+    pre {
+        background-color: #0f172a;
+        color: #7dd3fc;
+        padding: 1rem;
+        border-radius: 0.375rem;
+        border: 1px solid var(--bs-border-color);
+        overflow-x: auto;
+    }
+    /* Table styling */
+    .table {
+        color: #e2e8f0;
+    }
+    .table td {
+        border-color: var(--bs-border-color);
+    }
+    /* Updated text colors */
+    .text-primary {
+        color: white !important;
+    }
+    .text-muted {
+        color: #94a3b8 !important;
+    }
+    
+    /* Scrollable log list */
+    .log-list {
+        max-height: 400px;
+        overflow-y: auto;
+        background-color: #1e293b;
+        border-radius: 0.375rem;
+        border: 1px solid var(--bs-border-color);
+    }
+    .log-list::-webkit-scrollbar {
+        width: 8px;
+        background-color: #1e293b;
+    }
+    .log-list::-webkit-scrollbar-thumb {
+        background-color: #334155;
+        border-radius: 4px;
+    }
+
+    /* Ensure white text in code blocks and tables */
+    pre {
+        color: white !important;
+    }
+    .environment-table {
+        color: white !important;
+    }
+    
+    /* Hover states for log links */
+    .log-list a {
+        color: white !important;
+        text-decoration: none;
+    }
+    .log-list a:hover {
+        color: var(--bs-primary) !important;
+    }
+
+    /* Tab navigation styling */
+    .nav-tabs .nav-link {
+        color: white !important;
+        border-color: var(--bs-border-color);
+    }
+    .nav-tabs .nav-link:hover {
+        color: var(--bs-primary) !important;
+        border-color: var(--bs-border-color);
+    }
+    .nav-tabs .nav-link.active {
+        color: var(--bs-primary) !important;
+        background-color: #1e293b;
+        border-color: var(--bs-border-color);
+    }
+
+    /* Navigation text overflow prevention */
+    #test-case-nav a,
+    #error-list a {
+        white-space: normal !important;
+        word-wrap: break-word !important;
+        word-break: break-word !important;
+        padding: 8px 12px !important;
+        line-height: 1.4 !important;
+    }
+
+    /* Ensure proper spacing in nav sidebar */
+    .nav-sidebar .list-group-item {
+        border-radius: 0;
+        margin-bottom: 2px;
+    }
+
+    /* Common Errors list styling */
+    #error-list {
+        background-color: #1e293b;
+        border-radius: 0.375rem;
+        border: 1px solid var(--bs-border-color);
+        margin-bottom: 1rem;
+    }
+
+    /* Test Cases navigation styling */
+    #test-case-nav {
+        max-height: calc(100vh - 400px);
+        overflow-y: auto;
+    }
+</style>
+"""
+
+# rename all log files to remove colons
+def rename_log_files():
+    for log_file in FAILED_LOGS_DIR.glob("*.zip"):
+        safe_name = log_file.name.replace("::", "/")
+        dest_path = FAILED_LOGS_DIR / safe_name
+        dest_path.parent.mkdir(parents=True, exist_ok=True)
+        shutil.move(log_file, dest_path)
+
+
+def get_bazel_failures():
+    """Parse Bazel logs to find compilation failures."""
+    failed_tests = {}
+    log_files = Path(f"{BASE_DIR}/artifacts/bazel_event_logs/").glob("bazel_log*")
+    cnt = 0
+
+    for log_file in log_files:
+        try:
+            with open(log_file, "r") as f:
+                for line in f:
+                    event = json.loads(line)
+
+                    def search(label, value, res):
+                        for key, child in value.items():
+                            if key == label and isinstance(child, str):
+                                res.append(child)
+                            elif isinstance(child, dict):
+                                search(label, child, res)
+
+                    labels = []
+                    stderrs = []
+                    search("stderr", event, stderrs)
+                    search("label", event, labels)
+                    label = None
+                    if len(labels) > 0:
+                        label = labels[0]
+                    stderr = "\n".join(stderrs)
+                    if len(stderrs) > 0:
+                        if label is not None:
+                            failed_tests[label] = stderr
+                        elif "error generated" in stderr:
+                            failed_tests["Error " + str(cnt)] = "\n".join(stderrs)
+                            cnt += 1
+
+        except Exception as e:
+            print(f"Error reading {log_file}: {str(e)}")
+    return failed_tests
+
+
+def generate_html_report():
+    """Generate JSON results file only."""
+    REPORT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # Collect test results data
+    test_results = {
+        "commit_hash": COMMIT_HASH,
+        "label": REPORT_NAME,
+        "test_cases": [],
+        "error_counts": {},
+    }
+
+    # Process test summaries
+    test_summaries_dir = Path(f"{BASE_DIR}/artifacts/test-summaries/")
+    for idx, test_file in enumerate(test_summaries_dir.glob("*.txt")):
+        if "000_header.txt" in test_file.name:
+            continue
+
+        try:
+            content = test_file.read_text()
+            test_name = test_file.name.replace("bazel-out::", "").replace("::", "/")
+            pos = test_name.find("com_github_ray_project_ray")
+            test_name = test_name[pos + len("com_github_ray_project_ray/") :]
+
+            # Extract error pattern
+            error_pattern = "No error"
+            error_lines = [
+                line.strip() for line in content.splitlines() if line.startswith("E   ")
+            ]
+            if error_lines:
+                error_line = error_lines[-1]
+                error_pattern = re.sub(r"\b\w+Error\b", "", error_line).strip(" :")
+                if not error_pattern:
+                    error_pattern = error_line.split(":")[-1].strip()
+
+                test_results["error_counts"][error_pattern] = (
+                    test_results["error_counts"].get(error_pattern, 0) + 1
+                )
+
+            test_results["test_cases"].append(
+                {
+                    "id": f"test-case-{idx}",
+                    "name": test_name,
+                    "content": content,
+                    "error_pattern": error_pattern,
+                    "label": REPORT_NAME,
+                }
+            )
+
+        except Exception as e:
+            print(f"Error processing {test_file}: {str(e)}")
+            continue
+
+    # Process Bazel failures
+    bazel_failures = get_bazel_failures()
+    for idx, (label, content) in enumerate(bazel_failures.items()):
+        error_pattern = content.split("ERROR:")[0].strip()
+        test_results["error_counts"][error_pattern] = (
+            test_results["error_counts"].get(error_pattern, 0) + 1
+        )
+
+        test_results["test_cases"].append(
+            {
+                "id": f"bazel-failure-{idx}",
+                "name": label,
+                "content": content,
+                "error_pattern": error_pattern,
+                "label": REPORT_NAME,
+            }
+        )
+
+    # Save JSON results
+    json_path = REPORT_DIR / f"{REPORT_NAME}.json"
+    with open(json_path, "w") as f:
+        json.dump(test_results, f, indent=2)
+
+
+def move_generated_report():
+    """Copy log files to report directory."""
+    if not FAILED_LOGS_DIR.exists():
+        return
+    shutil.copytree(
+        FAILED_LOGS_DIR, REPORT_DIR / f"{REPORT_NAME}_logs", dirs_exist_ok=True
+    )
+
+
+def generate_summary_report(labels):
+    """Generate a summary report combining multiple test reports."""
+    all_results = []
+
+    # Load JSON results for each label
+    for label in labels:
+        json_path = REPORT_DIR / f"{label}.json"
+        if not json_path.exists():
+            print(f"Warning: Results for label {label} not found at {json_path}")
+            continue
+
+        with open(json_path) as f:
+            results = json.load(f)
+            # Ensure each test case has the correct label
+            for test_case in results["test_cases"]:
+                test_case["label"] = label
+            all_results.append(results)
+
+    # Combine results
+    combined_results = {
+        "commit_hash": COMMIT_HASH,
+        "test_cases": [],
+        "error_counts": {},
+    }
+
+    for results in all_results:
+        combined_results["test_cases"].extend(results["test_cases"])
+        for error, count in results["error_counts"].items():
+            combined_results["error_counts"][error] = (
+                combined_results["error_counts"].get(error, 0) + count
+            )
+
+    # Generate test cases HTML
+    test_cases_html = []
+    for test_case in combined_results["test_cases"]:
+        test_cases_html.append(
+            f"""
+            <div class="card test-card mb-4" id="{test_case['id']}" 
+                 data-error="{test_case['error_pattern']}"
+                 data-label="{test_case['label']}">
+                <div class="card-body">
+                    <h5 class="card-title text-danger">{test_case['name']}</h5>
+                    <div class="mb-3">
+                        <span class="badge bg-danger">Failed</span>
+                    </div>
+                    <pre>{test_case['content']}</pre>
+                </div>
+            </div>
+        """
+        )
+
+    # Generate HTML with navigation
+    html_content = f"""
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Summary Test Report</title>
+        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+        <style>
+            :root {{
+                --bs-body-bg: #f8fafc;
+                --bs-body-color: #1e293b;
+                --bs-primary: #3b82f6;
+            }}
+            
+            body {{ 
+                background-color: var(--bs-body-bg);
+                color: var(--bs-body-color);
+            }}
+            
+            .nav-top {{
+                position: sticky;
+                top: 0;
+                z-index: 1030;
+                background-color: white;
+                padding: 1rem;
+                border-bottom: 1px solid #e2e8f0;
+                box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+            }}
+            
+            .nav-top .nav-link {{
+                color: #64748b;
+                padding: 0.5rem 1rem;
+                margin: 0 0.25rem;
+                border-radius: 0.375rem;
+            }}
+            
+            .nav-top .nav-link:hover {{
+                background-color: #f1f5f9;
+                color: #1e293b;
+            }}
+            
+            .nav-top .nav-link.active {{
+                background-color: #3b82f6;
+                color: white;
+            }}
+            
+            .nav-sidebar {{
+                width: 280px;
+                position: fixed;
+                left: 0;
+                top: 72px;  /* Height of nav-top */
+                bottom: 0;
+                overflow-y: auto;
+                padding: 1rem;
+                background-color: white;
+                border-right: 1px solid #e2e8f0;
+            }}
+            
+            .main-content {{
+                margin-left: 280px;
+                padding: 2rem;
+                min-height: 100vh;
+            }}
+            
+            .card {{
+                border: none;
+                box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+                background-color: white;
+                border-radius: 0.5rem;
+                margin-bottom: 1rem;
+            }}
+            
+            .search-container {{
+                margin-bottom: 1.5rem;
+            }}
+            
+            pre {{
+                background-color: #f8fafc;
+                padding: 1rem;
+                border-radius: 0.375rem;
+                border: 1px solid #e2e8f0;
+            }}
+            
+            .list-group-item {{
+                border-color: #e2e8f0;
+            }}
+            
+            .list-group-item:hover {{
+                background-color: #f1f5f9;
+            }}
+        </style>
+    </head>
+    <body>
+        <!-- Top Navigation -->
+        <nav class="nav-top">
+            <div class="d-flex justify-content-between align-items-center">
+                <div class="d-flex align-items-center">
+                    <h4 class="me-4 mb-0">Test Reports</h4>
+                    <div class="nav nav-pills" id="test-tabs">
+                        {''.join(f'<a class="nav-link {label == labels[0] and "active" or ""}" data-label="{label}" href="#">{label}</a>' for label in labels)}
+                    </div>
+                </div>
+            </div>
+        </nav>
+
+        <!-- Sidebar with Common Errors -->
+        <nav class="nav-sidebar">
+            <h5 class="mb-3">Common Errors</h5>
+            <div class="list-group">
+                {generate_error_summaries_from_dict(combined_results['error_counts'])}
+            </div>
+        </nav>
+
+        <!-- Main Content -->
+        <main class="main-content">
+            <!-- Search Bar -->
+            <div class="search-container">
+                <input type="text" class="form-control" placeholder="Search test cases..." 
+                       id="searchInput" onkeyup="filterTests()">
+            </div>
+
+            <!-- Test Cases -->
+            <div id="test-cases-section">
+                {''.join(test_cases_html)}
+            </div>
+        </main>
+
+        <script>
+            let allTestCards = [];
+            const errorVisitState = {{}};
+            
+            document.addEventListener('DOMContentLoaded', () => {{
+                allTestCards = Array.from(document.querySelectorAll('.test-card'));
+                document.querySelectorAll('.error-item').forEach(initializeErrorItem);
+                
+                // Initialize tab navigation
+                document.querySelectorAll('#test-tabs .nav-link').forEach(tab => {{
+                    tab.addEventListener('click', (e) => {{
+                        e.preventDefault();
+                        
+                        // Update active state
+                        document.querySelectorAll('#test-tabs .nav-link').forEach(t => 
+                            t.classList.remove('active'));
+                        tab.classList.add('active');
+                        
+                        // Filter test cases by label
+                        const label = tab.getAttribute('data-label');
+                        filterTestsByLabel(label);
+                    }});
+                }});
+                
+                // Show initial label's test cases
+                filterTestsByLabel('{labels[0]}');
+            }});
+            
+            function filterTestsByLabel(label) {{
+                allTestCards.forEach(card => {{
+                    const cardLabel = card.getAttribute('data-label');
+                    card.style.display = (cardLabel === label) ? 'block' : 'none';
+                }});
+            }}
+            
+            function filterTests() {{
+                const filter = document.getElementById('searchInput').value.toLowerCase();
+                const activeLabel = document.querySelector('#test-tabs .nav-link.active')
+                    .getAttribute('data-label');
+                
+                allTestCards.forEach(card => {{
+                    const text = card.textContent.toLowerCase();
+                    const cardLabel = card.getAttribute('data-label');
+                    card.style.display = (text.includes(filter) && cardLabel === activeLabel) 
+                        ? 'block' : 'none';
+                }});
+            }}
+            
+            function initializeErrorItem(errorItem) {{
+                const errorPattern = errorItem.getAttribute('data-error');
+                const matchingCards = allTestCards.filter(
+                    card => card.getAttribute('data-error') === errorPattern
+                );
+                
+                errorItem.addEventListener('click', (e) => {{
+                    e.preventDefault();
+                    const state = errorVisitState[errorPattern] || {{ currentIndex: -1 }};
+                    state.currentIndex = (state.currentIndex + 1) % matchingCards.length;
+                    errorVisitState[errorPattern] = state;
+                    
+                    const targetCard = matchingCards[state.currentIndex];
+                    targetCard.scrollIntoView({{ behavior: 'smooth', block: 'start' }});
+                    
+                    const badge = errorItem.querySelector('.badge');
+                    if (badge) {{
+                        badge.textContent = `${{state.currentIndex + 1}}/${{matchingCards.length}}`;
+                    }}
+                }});
+            }}
+        </script>
+        
+        <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    </body>
+    </html>
+    """
+
+    summary_path = REPORT_DIR / "summary.html"
+    summary_path.write_text(dedent(html_content))
+    print(f"Summary report generated at: {summary_path}")
+
+
+def generate_error_summaries_from_dict(error_counts):
+    """Generate error summaries HTML from a dictionary of error counts."""
+    if not error_counts:
+        return ""
+
+    sorted_errors = sorted(error_counts.items(), key=lambda x: x[1], reverse=True)
+    return "\n".join(
+        f'<a href="#" class="list-group-item list-group-item-action error-item d-flex justify-content-between align-items-center py-2 text-truncate" data-error="{error}">'
+        f'<span class="text-truncate">{error}</span><span class="badge bg-danger ms-2">{count}</span></a>'
+        for error, count in sorted_errors
+    )
+
+
+def clean_base_dir():
+    for item in Path(BASE_DIR).iterdir():
+        if item.name == "reports":
+            continue
+        if item.is_file():
+            item.unlink()
+        elif item.is_dir():
+            shutil.rmtree(item)
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) > 1 and sys.argv[1] == "summary":
+        if len(sys.argv) < 3:
+            print("Error: Must provide labels when using summary option")
+            print("Usage: python report_gen.py summary label1 label2 ...")
+            sys.exit(1)
+        labels = sys.argv[2:]
+        generate_summary_report(labels)
+    else:
+        generate_html_report()
+        rename_log_files()
+        move_generated_report()
+        clean_base_dir()
+        print(f"Report generated at: {REPORT_DIR}/{REPORT_NAME}.json")
diff --git a/ci/ray_ci/tester.py b/ci/ray_ci/tester.py
index 732e4836671ce..8bf928120d7c8 100644
--- a/ci/ray_ci/tester.py
+++ b/ci/ray_ci/tester.py
@@ -203,7 +203,6 @@ def main(
         raise Exception("Please use `bazelisk run //ci/ray_ci`")
     os.chdir(bazel_workspace_dir)
     ci_init()
-    docker_login(_DOCKER_ECR_REPO.split("/")[0])
 
     if build_type == "wheel" or build_type == "wheel-aarch64":
         # for wheel testing, we first build the wheel and then use it for running tests
@@ -428,32 +427,4 @@ def _get_flaky_test_targets(
     """
     Get all test targets that are flaky
     """
-    if not yaml_dir:
-        yaml_dir = os.path.join(bazel_workspace_dir, "ci/ray_ci")
-
-    yaml_flaky_tests = set()
-    yaml_flaky_file = os.path.join(yaml_dir, f"{team}.tests.yml")
-    if os.path.exists(yaml_flaky_file):
-        with open(yaml_flaky_file, "rb") as f:
-            # load flaky tests from yaml
-            yaml_flaky_tests = set(yaml.safe_load(f)["flaky_tests"])
-
-    # load flaky tests from DB
-    s3_flaky_tests = {
-        # remove "linux:" prefix for linux tests to be consistent with the
-        # interface supported in the yaml file
-        test.get_name().lstrip("linux:")
-        for test in Test.gen_from_s3(prefix=f"{operating_system}:")
-        if test.get_oncall() == team and test.get_state() == TestState.FLAKY
-    }
-    all_flaky_tests = sorted(yaml_flaky_tests.union(s3_flaky_tests))
-
-    # linux tests are prefixed with "//"
-    if operating_system == "linux":
-        return [test for test in all_flaky_tests if test.startswith("//")]
-
-    # and other os tests are prefixed with "os:"
-    os_prefix = f"{operating_system}:"
-    return [
-        test.lstrip(os_prefix) for test in all_flaky_tests if test.startswith(os_prefix)
-    ]
+    return []
diff --git a/ci/ray_ci/tester_container.py b/ci/ray_ci/tester_container.py
index 204760bb11fb0..53dcd550542ee 100644
--- a/ci/ray_ci/tester_container.py
+++ b/ci/ray_ci/tester_container.py
@@ -107,8 +107,6 @@ def run_tests(
             for i in range(len(chunks))
         ]
         exits = [run.wait() for run in runs]
-        self._persist_test_results(team, bazel_log_dir_container, is_bisect_run)
-        self._cleanup_bazel_log_mount(bazel_log_dir_container)
 
         return all(exit == 0 for exit in exits)
 
diff --git a/ci/ray_ci/tests.env.Dockerfile b/ci/ray_ci/tests.env.Dockerfile
index b090dd6c07b0a..47b549398c09a 100644
--- a/ci/ray_ci/tests.env.Dockerfile
+++ b/ci/ray_ci/tests.env.Dockerfile
@@ -9,7 +9,7 @@ ARG BUILDKITE_CACHE_READONLY
 ENV CC=clang
 ENV CXX=clang++-12
 
-RUN mkdir /rayci
+RUN mkdir -p /rayci
 WORKDIR /rayci
 COPY . .
 
diff --git a/ci/run/bazel_export_options b/ci/run/bazel_export_options
index bbd7f3e2ab678..b41bb9588347d 100755
--- a/ci/run/bazel_export_options
+++ b/ci/run/bazel_export_options
@@ -1,15 +1,15 @@
 #!/usr/bin/env bash
 
-mkdir -p /tmp/bazel_event_logs
+mkdir -p "$(realpath "$BAZEL_DIR")"/bazel_event_logs
 if [[ "${OSTYPE}" == "msys" ]]; then
   artifact_mount_path="/c/artifact-mount"
 else
-  artifact_mount_path="/artifact-mount"
+  artifact_mount_path="$(realpath "$BAZEL_DIR")"
 fi
-event_json_flag=--build_event_json_file=$(mktemp /tmp/bazel_event_logs/bazel_log.XXXXX)
+event_json_flag=--build_event_json_file="$(mktemp "$(realpath "$BAZEL_DIR")"/bazel_event_logs/bazel_log.XXXXX)"
 logs_archive_flag=--test_env=RAY_TEST_FAILURE_LOGS_ARCHIVE_DIR="${artifact_mount_path}"/failed_test_logs
 summary_directory_flag=--test_env=RAY_TEST_SUMMARY_DIR="${artifact_mount_path}"/test-summaries
-summary_directory_host_flag=--test_env=RAY_TEST_SUMMARY_DIR_HOST=/tmp/artifacts/test-summaries
+summary_directory_host_flag=--test_env=RAY_TEST_SUMMARY_DIR_HOST="$(realpath "$BAZEL_DIR")"/artifacts/test-summaries
 buildkite_env_vars="--test_env=BUILDKITE_COMMIT --test_env=BUILDKITE_LABEL --test_env=BUILDKITE_JOB_ID"
 mujoco_env_var="--test_env=LD_LIBRARY_PATH=/root/.mujoco/mujoco210/bin"
 

From 6acb8028c56060086065586eaf955441a3bbc875 Mon Sep 17 00:00:00 2001
From: sule <sule.sl@antgroup.com>
Date: Tue, 25 Feb 2025 14:34:01 +0800
Subject: [PATCH 2/3] [CI] Add Ray ci

---
 .github/workflows/ray-ci.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ray-ci.yml b/.github/workflows/ray-ci.yml
index 6577348007edf..2c3e7fbf88a5e 100644
--- a/.github/workflows/ray-ci.yml
+++ b/.github/workflows/ray-ci.yml
@@ -94,6 +94,7 @@ jobs:
             sleep 60  # Wait 1 minute before trying again
           done
 
+
   check-permissions:
     name: Check User Permissions
     needs: start-ecs-runner
@@ -104,8 +105,9 @@ jobs:
       - name: Check if user is allowed
         id: check_user
         run: |
-          if [ -f "~/.workflow.cfg" ]; then
-            if grep -qE "^allowed_users=(${GITHUB_ACTOR}|.*,${GITHUB_ACTOR}|${GITHUB_ACTOR},.*|.*,${GITHUB_ACTOR},.*)\$" ~/.workflow.cfg; then
+          cat /home/runner/.workflow.cfg
+          if [ -f "/home/runner/.workflow.cfg" ]; then
+            if grep -qE "^allowed_users=(${GITHUB_ACTOR}|.*,${GITHUB_ACTOR}|${GITHUB_ACTOR},.*|.*,${GITHUB_ACTOR},.*)\$" /home/runner/.workflow.cfg; then
               echo "allowed=true" >> $GITHUB_OUTPUT
             else
               echo "allowed=false" >> $GITHUB_OUTPUT
@@ -165,6 +167,7 @@ jobs:
         else
           echo "exists=false" >> $GITHUB_OUTPUT
         fi
+
     - name: Build OSS CI Base
       if: steps.check_build_image.outputs.exists != 'true'
       uses: docker/build-push-action@v3

From 7a43756f32e367330cec96ade726bb4355784f4b Mon Sep 17 00:00:00 2001
From: sule <sule.sl@antgroup.com>
Date: Tue, 25 Feb 2025 14:50:36 +0800
Subject: [PATCH 3/3] make pr changes cancel old one

Signed-off-by: sule <sule.sl@antgroup.com>
---
 .github/workflows/ray-ci.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ray-ci.yml b/.github/workflows/ray-ci.yml
index 2c3e7fbf88a5e..f9ef616820923 100644
--- a/.github/workflows/ray-ci.yml
+++ b/.github/workflows/ray-ci.yml
@@ -11,6 +11,9 @@ env:
   DOCKER_BUILDKIT: 1
   RAY_CI_POST_WHEEL_TESTS: 1
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   start-ecs-runner:
@@ -18,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     environment: aliyun
     concurrency:
-      group: ${{ github.workflow }}-${{ github.ref }}
+      group: ${{ github.workflow }}
       cancel-in-progress: false
     outputs:
       leased_instance_id: ${{ steps.lease_instance.outputs.instance_id }}
@@ -96,8 +99,8 @@ jobs:
 
 
   check-permissions:
-    name: Check User Permissions
     needs: start-ecs-runner
+    name: Check User Permissions
     runs-on: ${{ needs.start-ecs-runner.outputs.leased_instance_id }}
     outputs:
       allowed: ${{ steps.check_user.outputs.allowed }}
@@ -179,7 +182,6 @@ jobs:
           DOCKER_IMAGE_BASE_TEST=ghcr.io/${{ github.repository }}/oss-ci-base_test-py${{ matrix.python }}
         push: true
 
-
   core-tests:
     name: Core Tests
     env: