unit_tests.yaml

name: unit-tests
on:
  push:
    branches:
      - "develop"
      - "main"
    paths-ignore:
      - "cmds/**"
      - "**.md"
  pull_request:
    branches:
      - "develop"
      - "main"
    paths-ignore:
      - "cmds/**"
      - "**.md"
env:
  WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4)
  SLURM_PARTITION: llm_s

jobs:
  unit_tests_core_pipeline:
    runs-on: [t_cluster]
    timeout-minutes: 20
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: core_pipeline
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_core/test_pipeline.py

  unit_tests_utils_storage_manager:
    runs-on: [t_cluster]
    timeout-minutes: 10
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: utils_storage_manager
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_utils/test_storage_manager.py

  unit_tests_model_fused_precision:
    runs-on: [t_cluster]
    timeout-minutes: 10
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: model_fused_precision
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_model/test_fused_precision/test_fused_precision.py

  unit_tests_data_batch_sampler:
    runs-on: [t_cluster]
    timeout-minutes: 20
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: data_batch_sample
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:8 python -m pytest -s -v ./tests/test_data/test_batch_sampler.py

  unit_tests_utils_timeout:
    runs-on: [t_cluster]
    timeout-minutes: 10
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: utils_timeout
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:1 python -m pytest -s -v ./tests/test_utils/test_timeout.py

  unit_tests_utils_model_checkpoint:
    runs-on: [t_cluster]
    timeout-minutes: 10
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - uses: actions/checkout@v3

    - name: utils_model_checkpoint
      run: |
        source $evo_env
        export PYTHONPATH=$PWD:$PYTHONPATH
        srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=internlm-ut-${GITHUB_RUN_ID}-${GITHUB_JOB} -N 1 -n 1 --gres=gpu:2 python -m pytest -s -v  ./tests/test_utils/test_model_checkpoint.py

  notify_to_feishu:
    if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }}
    needs: [
      unit_tests_core_pipeline,
      unit_tests_utils_storage_manager,
      unit_tests_model_fused_precision,
      unit_tests_data_batch_sampler,
      unit_tests_utils_timeout,
      unit_tests_utils_model_checkpoint
      ]
    runs-on: [t_cluster]
    steps:
    - name: mask env
      run: |
        echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
        echo "::add-mask::$path_prefix"
    - name: notify
      run: |
        curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"InternEvo GitHubAction Failed","content":[[{"tag":"text","text":""},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}'  ${{ secrets.WEBHOOK_URL }}