python-test.yml

name: Python package

on:
  push:
    branches: [master, release*]
  pull_request:
    branches: []
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  flake8-lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    name: Lint
    steps:
      - name: Check out source repository
        uses: actions/checkout@v2
      - name: Set up Python environment ${{ matrix.python-version }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python-version }}
      - name: flake8 Lint
        uses: py-actions/flake8@v1
        with:
          max-line-length: "120"
          exclude: "docs/samples/v1beta1/onnx/assets/*.py,python/kserve/kserve/*.py,python/kserve/test/test_v1*.py,python/kserve/kserve/__init__.py,python/kserve/test/__init__.py,python/kserve/test/test_knative*.py,python/kserve/kserve/protocol/grpc/grpc_predict_v2*.py"
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11"]
    steps:
      - name: Checkout source
        uses: actions/checkout@v2

      - name: Free-up disk space
        uses: ./.github/actions/free-up-disk-space

      - name: Set up Python ${{ matrix.python-version }}
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry and version plugin
        run: ./test/scripts/gh-actions/setup-poetry.sh

      - name: Configure Poetry cache
        run: |
          sudo mkdir -p /mnt/poetry
          # change permission so that poetry can install without sudo
          sudo chown -R $USER /mnt/poetry
          poetry config cache-dir /mnt/poetry

      # load poetry cache if cache exists
      - name: Load poetry cache
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: /mnt/poetry
          key: poetry-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      # ----------------------------------------Kserve Unit Tests--------------------------------------------------------
      # load cached kserve venv if cache exists
      - name: Load cached kserve venv
        id: cached-kserve-dependencies
        uses: actions/cache@v3
        with:
          path: python/kserve/.venv
          key: kserve-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock') }}
      # install kserve dependencies if cache does not exist
      - name: Install kserve dependencies
        if: steps.cached-kserve-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/kserve
          make install_dependencies
      - name: Install kserve
        run: |
          cd python/kserve
          make dev_install
      - name: Test kserve
        run: |
          cd python
          source kserve/.venv/bin/activate
          pytest --cov=kserve ./kserve

      # ----------------------------------------Kserve Pydantic V1 Unit Tests--------------------------------------------
      - name: Setup kserve pydantic v1 directory
        run: |
          mkdir -p python/kserve-pydantic-v1
          cp -r python/kserve/* python/kserve-pydantic-v1
          cd python/kserve-pydantic-v1
          # update the lock file without installing dependencies
          poetry update "pydantic<2.0" --lock
      - name: Load cached kserve pydantic v1 venv
        id: cached-kserve-pydantic-v1-dependencies
        uses: actions/cache@v3
        with:
          path: python/kserve-pydantic-v1/.venv
          key: kserve-pydantic-v1-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve-pydantic-v1/poetry.lock') }}
      # install kserve pydantic v1 dependencies if cache does not exist
      - name: Install kserve pydantic v1 dependencies
        if: steps.cached-kserve-pydantic-v1-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/kserve-pydantic-v1
          make install_dependencies
      - name: Install kserve pydantic v1
        run: |
          cd python/kserve-pydantic-v1
          make dev_install
      - name: Test kserve pydantic v1
        run: |
          cd python
          source kserve-pydantic-v1/.venv/bin/activate
          pytest --cov=kserve ./kserve-pydantic-v1

      # ----------------------------------------Sklearn Server Unit Tests------------------------------------------------
      # load cached sklearn venv if cache exists
      - name: Load cached sklearn venv
        id: cached-sklearn-dependencies
        uses: actions/cache@v3
        with:
          path: python/sklearnserver/.venv
          key: sklearn-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/sklearnserver/poetry.lock') }}
        # install sklearn server dependencies if cache does not exist
      - name: Install sklearn dependencies
        if: steps.cached-sklearn-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/sklearnserver
          make install_dependencies
      - name: Install sklearnserver
        run: |
          cd python/sklearnserver
          make dev_install
      - name: Test sklearnserver
        run: |
          cd python
          source sklearnserver/.venv/bin/activate
          pytest --cov=sklearnserver ./sklearnserver

      # ----------------------------------------Xgb Server Unit Tests------------------------------------------------
      # load cached xgb venv if cache exists
      - name: Load cached xgb venv
        id: cached-xgb-dependencies
        uses: actions/cache@v3
        with:
          path: python/xgbserver/.venv
          key: xgb-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/xgbserver/poetry.lock') }}
        # install xgb server dependencies if cache does not exist
      - name: Install xgb dependencies
        if: steps.cached-xgb-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/xgbserver
          make install_dependencies
      - name: Install xgbserver
        run: |
          cd python/xgbserver
          make dev_install
      - name: Test xgbserver
        run: |
          cd python
          source xgbserver/.venv/bin/activate
          pytest --cov=xgbserver ./xgbserver

      # ----------------------------------------Pmml Server Unit Tests------------------------------------------------
      # load cached pmml venv if cache exists
      - name: Load cached pmml venv
        id: cached-pmml-dependencies
        uses: actions/cache@v3
        with:
          path: python/pmmlserver/.venv
          key: pmml-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/pmmlserver/poetry.lock') }}
        # install pmml server dependencies if cache does not exist
      - name: Install pmml dependencies
        if: steps.cached-pmml-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/pmmlserver
          make install_dependencies
      - name: Install pmmlserver
        run: |
          cd python/pmmlserver
          make dev_install
      - name: Test pmmlserver
        run: |
          cd python
          source pmmlserver/.venv/bin/activate
          pytest --cov=pmmlserver ./pmmlserver

      # ----------------------------------------Lgb Server Unit Tests------------------------------------------------
      # load cached lgb venv if cache exists
      - name: Load cached lgb venv
        id: cached-lgb-dependencies
        uses: actions/cache@v3
        with:
          path: python/lgbserver/.venv
          key: lgb-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/lgbserver/poetry.lock') }}
        # install lgb server dependencies if cache does not exist
      - name: Install lgb dependencies
        if: steps.cached-lgb-dependencies.outputs.cache-hit != 'true'
        run: |
          cd python/lgbserver
          make install_dependencies
      - name: Install lgbserver
        run: |
          cd python/lgbserver
          make dev_install
      - name: Test lgbserver
        run: |
          cd python
          source lgbserver/.venv/bin/activate
          pytest --cov=lgbserver ./lgbserver

      # ----------------------------------------Paddle Server Unit Tests------------------------------------------------
      # load cached paddle venv if cache exists
      - name: Load cached paddle venv
        id: cached-paddle-dependencies
        uses: actions/cache@v3
        with:
          path: python/paddleserver/.venv
          key: paddle-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/paddleserver/poetry.lock') }}
        # install paddle server dependencies if cache does not exist
      - name: Install paddle dependencies
        if: steps.cached-paddle-dependencies.outputs.cache-hit != 'true'
        run: |
          echo "python version ${{ steps.setup-python.outputs.python-version }}"
          cd python/paddleserver
          make install_dependencies
      - name: Install paddleserver
        run: |
          cd python/paddleserver
          make dev_install
      - name: Test paddleserver
        run: |
          cd python
          source paddleserver/.venv/bin/activate
          pytest --cov=paddleserver ./paddleserver

      # ----------------------------------------Alibi Explainer Unit Tests------------------------------------------------
      # load cached alibi venv if cache exists
      - name: Load cached alibi venv
        id: cached-alibi-dependencies
        uses: actions/cache@v3
        with:
          path: python/alibiexplainer/.venv
          key: alibi-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/alibiexplainer/poetry.lock') }}
        # install alibi explainer dependencies if cache does not exist
      - name: Install alibi dependencies
        run: |
          cd python/alibiexplainer
          make install_dependencies
      - name: Install alibiexplainer
        run: |
          cd python/alibiexplainer
          make dev_install
      - name: Test alibiexplainer
        run: |
          cd python
          source alibiexplainer/.venv/bin/activate
          pytest --cov=alibiexplainer ./alibiexplainer

      # ----------------------------------------Huggingface Server Unit Tests------------------------------------------------
      # load cached huggingface venv if cache exists
      - name: Load cached huggingface venv
        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
        id: huggingface-dependencies
        uses: actions/cache@v3
        with:
          path: /mnt/python/huggingfaceserver-venv
          key: huggingface-venv-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/kserve/poetry.lock', '**/huggingfaceserver/poetry.lock') }}
        # install huggingface server dependencies if cache does not exist
      - name: Configure poetry for huggingface server
        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
        run: |
          poetry config virtualenvs.path /mnt/python/huggingfaceserver-venv
          poetry config virtualenvs.in-project false
      - name: Install huggingface dependencies
        if: ${{ steps.cached-huggingface-dependencies.outputs.cache-hit != 'true' && !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
        run: |
          sudo mkdir -p /mnt/python/huggingfaceserver-venv
          # change permission so that poetry can install without sudo
          sudo chown -R $USER /mnt/python/huggingfaceserver-venv
          cd python/huggingfaceserver
          make install_dependencies
      - name: Install huggingface server
        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
        run: |
          cd python/huggingfaceserver
          make dev_install
      - name: Test huggingfaceserver
        if: ${{ !startsWith(steps.setup-python.outputs.python-version, '3.8') }}
        run: |
          cd python/huggingfaceserver
          poetry run -- pytest --cov=huggingfaceserver

      - name: Free space after tests
        run: |
          df -hT