-
Notifications
You must be signed in to change notification settings - Fork 26.7k
225 lines (195 loc) · 8.35 KB
/
self-comment-slow-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
name: PR comment github action tests
on:
workflow_dispatch:
branches-ignore:
- main
issue_comment:
types:
- created
- labeled
branches-ignore:
- main
pull_request_review_comment:
types: [created, deleted]
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
HF_HOME: /mnt/cache
TRANSFORMERS_IS_CI: yes
OMP_NUM_THREADS: 8
MKL_NUM_THREADS: 8
RUN_SLOW: yes
# For gated repositories, we still need to agree to share information on the Hub repo. page in order to get access.
# This token is created under the bot `hf-transformers-bot`.
HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }}
SIGOPT_API_TOKEN: ${{ secrets.SIGOPT_API_TOKEN }}
TF_FORCE_GPU_ALLOW_GROWTH: true
RUN_PT_TF_CROSS_TESTS: 1
CUDA_VISIBLE_DEVICES: 0,1
jobs:
reply-and-trigger:
if: contains(github.event.comment.body,'/pytest')
runs-on: ubuntu-latest
steps:
- name: Set up GitHub CLI
uses: actions/checkout@v4
- name: Reply to the comment
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api --method POST \
/repos/${{ github.repository }}/pulls/comments/${{ github.event.comment.id }} \
-f body='This comment contains /pytest, running the specified job...'
- name: Trigger specific job
run: |
echo "Running the specific job because /pytest was found"
# Add the logic to trigger your specific job here
# This could be a script, another action, etc.
run_if_label_matches:
if: github.event.label.name == 'run-slow'
runs-on: ubuntu-latest
steps:
- run: echo 'The label was run-slow'
pr_commented:
# This job only runs for pull request comments
name: PR comment
if: ${{ github.event.issue.pull_request }}
runs-on: ubuntu-latest
steps:
- run: |
echo A comment on PR $NUMBER
env:
NUMBER: ${{ github.event.issue.number }}
testing-stuff-around:
if: contains(github.event.comment.html_url, '/pull/') # check if the comments come from pull request, exclude those from issue.
runs-on: ubuntu-22.04
steps:
- name: say hello
if: contains(github.event.comment.body, '/pytest') # check the comment if it contains the keywords
run: |
echo say hello
- name: test
if: contains(github.event.comment.body, '/test') # check the comment if it contains the keywords
run: |
echo test
find_models_to_run:
runs-on: ubuntu-22.04
name: Find models to run slow tests
# Triggered only if the required label `run-slow` is added
if: github.event.issue.pull_request && contains(github.event.comment.body,'/pytest')
outputs:
models: ${{ steps.models_to_run.outputs.models }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: "0"
ref: ${{ github.event.pull_request.head.sha }}
- name: Get commit message
run: |
echo "commit_message=$(git show -s --format=%s)" >> $GITHUB_ENV
- name: Parse comment and set machine type
if: github.event_name == 'issue_comment'
run: |
COMMENT=${{ github.event.comment.body }}
if [[ "$COMMENT" == *"--device"* ]]; then
DEVICE=$(echo "$COMMENT" | grep -oP '(?<=--device\s)\S+')
echo "machine_type=${DEVICE}" >> $GITHUB_OUTPUT
fi
- name: Get models to run slow tests
run: |
echo "${{ env.commit_message }}"
python -m pip install GitPython
python utils/pr_slow_ci_models.py --commit_message "${{ env.commit_message }}" | tee output.txt
echo "models=$(tail -n 1 output.txt)" >> $GITHUB_ENV
- name: Models to run slow tests
id: models_to_run
run: |
echo "${{ env.models }}"
echo "models=${{ env.models }}" >> $GITHUB_OUTPUT
run_models_gpu:
name: Run all tests for the model
if: ${{ needs.find_models_to_run.outputs.models != '[]' || github.event_name == 'issue_comment' }}
needs: find_models_to_run
strategy:
fail-fast: false
matrix:
folders: ${{ fromJson(needs.find_models_to_run.outputs.models) || '[]' }}
machine_type: [single-gpu, multi-gpu]
runs-on: ['${{ matrix.machine_type }}', nvidia-gpu, t4, ci]
container:
image: huggingface/transformers-all-latest-gpu
options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
steps:
- name: Parse command from PR comment
if: ${{ github.event_name == 'issue_comment' }}
id: parse_command
run: |
comment="${{ github.event.comment.body }}"
if [[ "$comment" =~ /run[[:space:]]+pytest ]]; then
export pytest_command=$(echo "$comment" | sed -n 's/\/run[[:space:]]\+pytest[[:space:]]\+\(.*\)/\1/p')
echo "pytest_command=$pytest_command" >> $GITHUB_ENV
else
echo "Invalid command. Exiting."
exit 1
env:
pytest_command: ""
- name: Echo input and matrix info
if: ${{ github.event_name == 'pull_request' || env.pytest_command != '' }}
shell: bash
run: |
echo "${{ matrix.folders }}"
- name: Echo folder ${{ matrix.folders }}
if: ${{ github.event_name == 'pull_request' || env.pytest_command != '' }}
shell: bash
run: |
echo "${{ matrix.folders }}"
matrix_folders=${{ matrix.folders }}
matrix_folders=${matrix_folders/'models/'/'models_'}
echo "$matrix_folders"
echo "matrix_folders=$matrix_folders" >> $GITHUB_ENV
- name: Update clone
working-directory: /transformers
run: git fetch && git fetch origin pull/${{ github.event.pull_request.number }}/head:pull/${{ github.event.pull_request.number }}/merge && git checkout pull/${{ github.event.pull_request.number }}/merge
- name: Reinstall transformers in edit mode (remove the one installed during docker image build)
working-directory: /transformers
run: python3 -m pip uninstall -y transformers && python3 -m pip install -e .
- name: NVIDIA-SMI
run: |
nvidia-smi
- name: Environment
working-directory: /transformers
run: |
python3 utils/print_env.py
- name: Show installed libraries and their versions
working-directory: /transformers
run: pip freeze
- name: Run all tests on GPU
working-directory: /transformers
run: |
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py --test_folder ${{ matrix.folders }})"
echo $CUDA_VISIBLE_DEVICES
python3 -m pytest -v -rsfE --make-reports=${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports tests/${{ matrix.folders }}
- name: Run custom tests from comment
working-directory: /transformers
if: ${{ github.event_name == 'issue_comment' && env.pytest_command != '' }}
run: |
export CUDA_VISIBLE_DEVICES="$(python3 utils/set_cuda_devices_for_ci.py)"
echo $CUDA_VISIBLE_DEVICES
python3 -m pytest -v ${{ env.pytest_command }}
- name: Failure short reports
if: ${{ failure() }}
continue-on-error: true
run: cat /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/failures_short.txt
- name: Make sure report directory exists
shell: bash
run: |
mkdir -p /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports
echo "hello" > /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports/hello.txt
echo "${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports"
- name: "Test suite reports artifacts: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports"
if: ${{ always() }}
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.machine_type }}_run_models_gpu_${{ env.matrix_folders }}_test_reports
path: /transformers/reports/${{ matrix.machine_type }}_run_models_gpu_${{ matrix.folders }}_test_reports