Package test #308
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Package test | |
| on: | |
| workflow_dispatch: | |
| # Schedule the workflow to run at 08:00 (UTC) every day. | |
| schedule: | |
| # Minute[0,59] Hour[0,23] Day of month[1,31] Month[1,12] Day of week[0,6] (Sunday=0) | |
| - cron: '0 8 * * *' | |
| push: | |
| paths: | |
| - "scalellm/**" | |
| - "tests/**" | |
| - "setup.py" | |
| - "requirements.txt" | |
| - "requirements-test.txt" | |
| - ".github/workflows/package_test.yml" | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - "scalellm/**" | |
| - "tests/**" | |
| - "setup.py" | |
| - "requirements.txt" | |
| - "requirements-test.txt" | |
| - ".github/workflows/package_test.yml" | |
| branches: | |
| - main | |
| env: | |
| # Tells where to store caches. | |
| CI_CACHE_DIR: ${{ github.workspace }}/../../ci_cache | |
| # cancel all previous runs if a new one is triggered | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-and-test-wheel: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python: ["3.10"] | |
| cuda: ["12.4"] | |
| torch: ["2.4.1"] | |
| runs-on: [self-hosted, linux, build] | |
| env: | |
| PYTHON_VERSION: ${{ matrix.python }} | |
| CUDA_VERSION: ${{ matrix.cuda }} | |
| TORCH_VERSION: ${{ matrix.torch }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Create cache directory | |
| run: | | |
| mkdir -p $CI_CACHE_DIR/.vcpkg/bincache | |
| mkdir -p $CI_CACHE_DIR/.ccache | |
| mkdir -p $CI_CACHE_DIR/.pip | |
| - name: Build wheel | |
| timeout-minutes: 60 | |
| run: | | |
| docker pull vectorchai/scalellm_manylinux:cuda${CUDA_VERSION} | |
| docker run --rm -t \ | |
| -v "$CI_CACHE_DIR":/ci_cache \ | |
| -v "$GITHUB_WORKSPACE":/ScaleLLM \ | |
| -e PYTHON_VERSION=${PYTHON_VERSION} \ | |
| -e CUDA_VERSION=${CUDA_VERSION} \ | |
| -e TORCH_VERSION=${TORCH_VERSION} \ | |
| -e VCPKG_DEFAULT_BINARY_CACHE=/ci_cache/.vcpkg/bincache \ | |
| -e CCACHE_DIR=/ci_cache/.ccache \ | |
| -e PIP_CACHE_DIR=/ci_cache/.pip \ | |
| -u $(id -u):$(id -g) \ | |
| vectorchai/scalellm_manylinux:cuda${CUDA_VERSION} \ | |
| bash /ScaleLLM/scripts/build_wheel.sh | |
| - name: Show whl package size | |
| run: du -h dist/* | |
| - name: Install the package and run pytest | |
| timeout-minutes: 10 | |
| shell: bash -l {0} | |
| run: | | |
| # define the environment name based on the python and torch versions | |
| ENV_NAME=pkg_test_py${PYTHON_VERSION//./}_cu${CUDA_VERSION//./}_torch${TORCH_VERSION//./} | |
| # Initialize conda environment | |
| source ~/anaconda3/etc/profile.d/conda.sh | |
| conda init --all && source ~/.bashrc | |
| # Create a new conda environment | |
| if conda info --envs | grep -q ${ENV_NAME}; then | |
| echo "${ENV_NAME} already exists" | |
| else | |
| conda create -y -n ${ENV_NAME} python=${PYTHON_VERSION}; | |
| fi | |
| # Activate the conda environment | |
| conda activate ${ENV_NAME} | |
| # Install PyTorch | |
| pip install torch==${TORCH_VERSION} -i "https://download.pytorch.org/whl/cu${CUDA_VERSION//./}" | |
| # Uninstall previous version of the package and install the new one | |
| pip uninstall -y scalellm | |
| pip install dist/*.whl | |
| # Install the test requirements | |
| pip install -r requirements-test.txt | |
| # Run pytest | |
| printf "\n\nRunning pytest\n\n" | |
| cd tests | |
| python3 -m pytest || exit 1 | |
| printf "\n\n" | |
| # Deactivate the conda environment | |
| conda deactivate || true |