test(ci): optimize ci (#27) #126
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: unit-tests | |
on: | |
push: | |
branches: | |
- "develop" | |
- "main" | |
paths-ignore: | |
- "cmds/**" | |
- "**.md" | |
pull_request: | |
branches: | |
- "develop" | |
- "main" | |
paths-ignore: | |
- "cmds/**" | |
- "**.md" | |
env: | |
WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4) | |
SLURM_PARTITION: llm_s | |
jobs: | |
unit_tests_core_pipeline: | |
runs-on: [t_cluster] | |
timeout-minutes: 20 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: core_pipeline | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_core/test_pipeline.py | |
unit_tests_utils_storage_manager: | |
runs-on: [t_cluster] | |
timeout-minutes: 10 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: utils_storage_manager | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_utils/test_storage_manager.py | |
unit_tests_model_fused_precision: | |
runs-on: [t_cluster] | |
timeout-minutes: 10 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: model_fused_precision | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_model/test_fused_precision/test_fused_precision.py | |
unit_tests_data_batch_sampler: | |
runs-on: [t_cluster] | |
timeout-minutes: 20 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: data_batch_sample | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_data/test_batch_sampler.py | |
unit_tests_utils_timeout: | |
runs-on: [t_cluster] | |
timeout-minutes: 10 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: utils_timeout | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:1 python -m pytest -s -v ./tests/test_utils/test_timeout.py | |
unit_tests_utils_model_checkpoint: | |
runs-on: [t_cluster] | |
timeout-minutes: 10 | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- uses: actions/checkout@v3 | |
- name: utils_model_checkpoint | |
run: | | |
source $evo_env | |
export PYTHONPATH=$PWD:$PYTHONPATH | |
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:2 python -m pytest -s -v ./tests/test_utils/test_model_checkpoint.py | |
notify_to_feishu: | |
if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }} | |
needs: [ | |
unit_tests_core_pipeline, | |
unit_tests_utils_storage_manager, | |
unit_tests_model_fused_precision, | |
unit_tests_data_batch_sampler, | |
unit_tests_utils_timeout, | |
unit_tests_utils_model_checkpoint | |
] | |
runs-on: [t_cluster] | |
steps: | |
- name: mask env | |
run: | | |
echo "::add-mask::${{env.WORKSPACE_PREFIX}}" | |
echo "::add-mask::$path_prefix" | |
- name: notify | |
run: | | |
curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"InternEvo GitHubAction Failed","content":[[{"tag":"text","text":""},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}' ${{ secrets.WEBHOOK_URL }} |