Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@ydshieh
ydshieh committed Oct 7, 2024
1 parent 9e9421e commit adfbca7
Showing 1 changed file with 152 additions and 136 deletions.
288 changes: 152 additions & 136 deletions .github/workflows/self-comment-slow-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,40 +4,17 @@ on:
issue_comment:
types:
- created
- labeled
branches-ignore:
- main
pull_request_review_comment:
types: [created, deleted]
pull_request:
# pull_request:
concurrency:
# But this will cancel the current job no ? Because they have the same PR number ... (any comment even not the target one)
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}-${{ contains(github.event.comment.body, 'run-slow') }}
cancel-in-progress: ${{ (github.event.issue.pull_request != null || github.event.pull_request != null) }}

jobs:
reply-and-trigger:
runs-on: ubuntu-latest
permissions: write-all
steps:
- name: Trigger specific job
run: |
echo "${{ github.event.comment.body }}"
echo "${{ startsWith(github.event.comment.body, 'run-slow') }}"
echo "hello 4"
echo "${{ github.head_ref }}"
echo "${{ github.ref }}"
echo "${{ github.sha }}"
echo "${{ github.event.pull_request.number }}"
echo "${{ github.event.issue.number }}"
echo "${{ github.event.issue.pull_request }}"
echo "${{ github.event.pull_request }}"
echo "${{ github.event.issue.pull_request != null }}"
echo "${{ github.event.pull_request != null }}"
echo "${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}-${{ contains(github.event.comment.body, 'run-slow') }}"
echo "${{ github.event.pull_request.head.sha }}"
echo "${{ github.event.pull_request.merge_commit_sha }}"
get-pr-number:
runs-on: ubuntu-22.04
name: Get PR number
Expand Down Expand Up @@ -67,34 +44,6 @@ jobs:
id: set_pr_number
run: echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> "$GITHUB_OUTPUT"

check-pr-number:
runs-on: ubuntu-latest
needs: get-pr-number
steps:
- env:
PR_NUMBER: ${{needs.get-pr-number.outputs.PR_NUMBER}}
run: |
echo "$PR_NUMBER"
echo "${{ env.PR_NUMBER }}"
check-pr-merge-branch:
runs-on: ubuntu-latest
needs: get-pr-number
steps:
- uses: actions/checkout@v4
with:
fetch-depth: "0"
ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"

- name: Checkout PR merge branch
env:
PR_NUMBER: ${{needs.get-pr-number.outputs.PR_NUMBER}}
run: |
git fetch
git fetch origin refs/pull/$PR_NUMBER/merge:refs/remotes/pull/$PR_NUMBER/merge
git checkout refs/remotes/pull/$PR_NUMBER/merge
git log -n 5
# use a python script to handle this complex logic
# case 1: `run-slow` (auto. infer with limited number of models, but in particular, new model)
# case 2: `run-slow model_1, model_2`
Expand All @@ -103,6 +52,7 @@ jobs:
runs-on: ubuntu-latest
needs: get-pr-number
if: ${{ needs.get-pr-number.outputs.PR_NUMBER != ''}}
permissions: write-all
outputs:
models: ${{ steps.models_to_run.outputs.models }}
steps:
Expand All @@ -111,6 +61,17 @@ jobs:
fetch-depth: "0"
ref: "refs/pull/${{needs.get-pr-number.outputs.PR_NUMBER}}/merge"

- name: Reply to the comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
repos/${{ github.repository }}/issues/${{ needs.get-pr-number.outputs.PR_NUMBER }}/comments \
-f "body=This comment contains run-slow, running the specified job..."
- name: Get models to test
run: |
python -m pip install GitPython
Expand All @@ -123,27 +84,55 @@ jobs:
echo "${{ env.models }}"
echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
show-tests:
runs-on: ubuntu-latest
name: Show model to test
create_run:
name: Create run
if: ${{ needs.get-tests.outputs.models != '[]' }}
needs: get-tests
permissions: write-all
runs-on: ubuntu-latest
steps:
- name: Show models to test
- name: Create Run
id: create_run
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
echo "${{ needs.get-tests.outputs.models }}"
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
-f "target_url=$GITHUB_RUN_URL" -f "state=pending" -f "description=Slow CI job" -f "context=pytest/custom-tests"
run_models_gpu_dummy:
name: Run all tests for the model
# Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
# (either a new model PR or via a commit message)
if: ${{ needs.get-tests.outputs.models != '[]' }}
needs: [get-tests, create_run]
strategy:
fail-fast: false
matrix:
folders: ${{ fromJson(needs.get-tests.outputs.models) }}
machine_type: [aws-g4dn-2xlarge-cache, aws-g4dn-12xlarge-cache]
runs-on: ubuntu-latest
steps:
- name: Echo input and matrix info
shell: bash
run: |
echo "${{ matrix.folders }}"
run_models_gpu:
name: Run all tests for the model
# Triggered only `find_models_to_run` is triggered (label `run-slow` is added) which gives the models to run
# (either a new model PR or via a commit message)
if: ${{ needs.get-tests.outputs.models != '[]' }}
needs: get-tests
needs: [get-tests, create_run]
strategy:
fail-fast: false
matrix:
folders: ${{ fromJson(needs.get-tests.outputs.models) }}
machine_type: [aws-g4dn-2xlarge-cache, aws-g4dn-12xlarge-cache]
# runs-on: ubuntu-latest
runs-on:
group: '${{ matrix.machine_type }}'
container:
Expand All @@ -155,80 +144,107 @@ jobs:
run: |
echo "${{ matrix.folders }}"
# git-checkout:
# runs-on: ubuntu-22.04
# name: Check PR merge branch
# steps:
# - name: Get PR number
# run: |
# if
#
# - uses: actions/checkout@v4
# with:
# fetch-depth: "0"
# # ref: ${{ github.event.pull_request.head.sha }}
# ref: "refs/pull/${{ github.event.pull_request.number }}/merge"
#
# - name: Checkout
# run: |
# git fetch
# git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge
# git checkout pull/${{ github.event.pull_request.number }}/merge
# git log -n 5
#
# - name: Checkout 2
# run: |
# git fetch
# git fetch origin refs/pull/${{ github.event.pull_request.number }}/merge:refs/remotes/pull/${{ github.event.pull_request.number }}/merge
# git checkout refs/remotes/pull/${{ github.event.pull_request.number }}/merge
# git log -n 5
#
# - name: Checkout 3
# run: |
# git fetch
# git fetch origin refs/pull/${{ github.event.pull_request.number }}/head:refs/remotes/pull/${{ github.event.pull_request.number }}/head
# git checkout refs/remotes/pull/${{ github.event.pull_request.number }}/head
# git log -n 6

# - name: Set up GitHub CLI
# uses: actions/checkout@v4
#
# - name: Reply to the comment
# env:
# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# run: |
# gh api \
# --method POST \
# -H "Accept: application/vnd.github+json" \
# -H "X-GitHub-Api-Version: 2022-11-28" \
# /repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
# -f "body=This comment contains /pytest, running the specified job..."
#
# - name: Create Run
# id: create_run
# env:
# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
# run: |
# gh api \
# --method POST \
# -H "Accept: application/vnd.github+json" \
# -H "X-GitHub-Api-Version: 2022-11-28" \
# /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
# -f "target_url=$GITHUB_RUN_URL" -f "state=pending" -f "description=Custom CI job" -f "context=pytest/custom-tests"
# - name: Trigger specific job
# run: |
# echo "Running the specific job because /pytest was found"
# # Add the logic to trigger your specific job here

# - name: Update Check Run Status
# env:
# GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
# run: |
# gh api \
# --method POST \
# -H "Accept: application/vnd.github+json" \
# -H "X-GitHub-Api-Version: 2022-11-28" \
# /repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
# -f "target_url=$GITHUB_RUN_URL" -f "state=success" -f "description=Custom CI job" -f "context=pytest/custom-tests"
- name: Echo folder ${{ matrix.folders }}
shell: bash
# For folders like `models/bert`, set an env. var. (`matrix_folders`) to `models_bert`, which will be used to
# set the artifact folder names (because the character `/` is not allowed).
run: |
echo "${{ matrix.folders }}"
matrix_folders=${{ matrix.folders }}
matrix_folders=${matrix_folders/'models/'/'models_'}
echo "$matrix_folders"
echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
- name: Update clone
working-directory: /transformers
run: git fetch && git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge && git checkout pull/${{ github.event.pull_request.number }}/merge

- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
working-directory: /transformers
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .

- name: NVIDIA-SMI
run: |
nvidia-smi
- name: Set `machine_type` for report and artifact names
working-directory: /transformers
shell: bash
run: |
echo "${{ matrix.machine_type }}"
if [ "${{ matrix.machine_type }}" = "aws-g4dn-2xlarge-cache" ]; then
machine_type=single-gpu
elif [ "${{ matrix.machine_type }}" = "aws-g4dn-12xlarge-cache" ]; then
machine_type=multi-gpu
else
machine_type=${{ matrix.machine_type }}
fi
echo "$machine_type"
echo "machine_type=$machine_type" >> $GITHUB_ENV
- name: Environment
working-directory: /transformers
run: |
python3 utils/print_env.py
- name: Show installed libraries and their versions
working-directory: /transformers
run: pip freeze

- name: Run all tests on GPU
working-directory: /transformers
run: |
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
echo $CUDA_VISIBLE_DEVICES
python3 -m pytest -v -rsfE --make-reports=${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Failure short reports
if: ${{ failure() }}
continue-on-error: true
run: cat /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt

- name: Make sure report directory exists
shell: bash
run: |
mkdir -p /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
echo "hello" > /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
echo "${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
- name: "Test suite reports artifacts: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: ${{ env.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ env.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports

update_run_status:
name: Update Check Run Status
needs: [run_models_gpu]
permissions: write-all
if: always()
runs-on: ubuntu-latest
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
steps:
- name: Get `run_models_gpu` job status
run: |
echo "${{ needs.run_models_gpu.result }}"
if [ "${{ needs.run_models_gpu.result }}" = "cancelled" ]; then
echo "STATUS=failure" >> $GITHUB_ENV
elif [ "${{ needs.run_models_gpu.result }}" = "skipped" ]; then
echo "STATUS=success" >> $GITHUB_ENV
else
echo "STATUS=${{ needs.run_models_gpu.result }}" >> $GITHUB_ENV
fi
- name: Update run's status
run: |
echo "${{ needs.run_models_gpu.result }}"
echo "${{ env.STATUS }}"
gh api \
--method POST \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
-f "target_url=$GITHUB_RUN_URL" -f "state=${{ env.STATUS }}" -f "description=Slow CI job" -f "context=pytest/custom-tests"

0 comments on commit adfbca7

Please sign in to comment.