tests.yml

name: Tests
on:
  push:
    branches:
      - develop
  pull_request:
    branches-ignore:
      - master
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

env:
  DEFAULT_PYTHON: '3.12'

jobs:
  get-matrix:
    name: Get version matrix based on changes in PR/commit
    uses: ./.github/workflows/get-matrix.yml

  tests-core:
    name: Run core tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-core) }}

    uses: ./.github/workflows/test-core.yml
    with:
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-clickhouse:
    name: Run Clickhouse tests (server=${{ matrix.clickhouse-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-clickhouse) }}

    uses: ./.github/workflows/test-clickhouse.yml
    with:
      clickhouse-image: ${{ matrix.clickhouse-image }}
      clickhouse-version: ${{ matrix.clickhouse-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-greenplum:
    name: Run Greenplum tests (server=${{ matrix.greenplum-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-greenplum) }}

    uses: ./.github/workflows/test-greenplum.yml
    with:
      greenplum-version: ${{ matrix.greenplum-version }}
      spark-version: ${{ matrix.spark-version }}
      package-version: ${{ matrix.package-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}
    secrets:
      GREENPLUM_PACKAGES_USER: ${{ secrets.GREENPLUM_PACKAGES_USER }}
      GREENPLUM_PACKAGES_PASSWORD: ${{ secrets.GREENPLUM_PACKAGES_PASSWORD }}

  tests-hive:
    name: Run Hive tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-hive) }}

    uses: ./.github/workflows/test-hive.yml
    with:
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-kafka:
    name: Run Kafka tests (server=${{ matrix.kafka-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-kafka) }}

    uses: ./.github/workflows/test-kafka.yml
    with:
      kafka-version: ${{ matrix.kafka-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-local-fs:
    name: Run LocalFS tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-local-fs) }}

    uses: ./.github/workflows/test-local-fs.yml
    with:
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-mongodb:
    name: Run MongoDB tests (server=${{ matrix.mongodb-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-mongodb) }}

    uses: ./.github/workflows/test-mongodb.yml
    with:
      mongodb-version: ${{ matrix.mongodb-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-mssql:
    name: Run MSSQL tests (server=${{ matrix.mssql-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-mssql) }}

    uses: ./.github/workflows/test-mssql.yml
    with:
      mssql-version: ${{ matrix.mssql-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-mysql:
    name: Run MySQL tests (server=${{ matrix.mysql-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-mysql) }}

    uses: ./.github/workflows/test-mysql.yml
    with:
      mysql-version: ${{ matrix.mysql-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-oracle:
    name: Run Oracle tests (server=${{ matrix.oracle-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-oracle) }}

    uses: ./.github/workflows/test-oracle.yml
    with:
      oracle-image: ${{ matrix.oracle-image }}
      oracle-version: ${{ matrix.oracle-version }}
      db-name: ${{ matrix.db-name }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-postgres:
    name: Run Postgres tests (server=${{ matrix.postgres-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-postgres) }}

    uses: ./.github/workflows/test-postgres.yml
    with:
      postgres-version: ${{ matrix.postgres-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-teradata:
    name: Run Teradata tests (spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-teradata) }}

    uses: ./.github/workflows/test-teradata.yml
    with:
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-ftp:
    name: Run FTP tests (server=${{ matrix.ftp-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-ftp) }}

    uses: ./.github/workflows/test-ftp.yml
    with:
      ftp-version: ${{ matrix.ftp-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-ftps:
    name: Run FTPS tests (server=${{ matrix.ftps-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-ftps) }}

    uses: ./.github/workflows/test-ftps.yml
    with:
      ftps-version: ${{ matrix.ftps-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-hdfs:
    name: Run HDFS tests (server=${{ matrix.hadoop-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-hdfs) }}

    uses: ./.github/workflows/test-hdfs.yml
    with:
      hadoop-version: ${{ matrix.hadoop-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-s3:
    name: Run S3 tests (server=${{ matrix.minio-version }}, spark=${{ matrix.spark-version }}, pydantic=${{ matrix.pydantic-version }}, java=${{ matrix.java-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-s3) }}

    uses: ./.github/workflows/test-s3.yml
    with:
      minio-version: ${{ matrix.minio-version }}
      spark-version: ${{ matrix.spark-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      java-version: ${{ matrix.java-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-sftp:
    name: Run SFTP tests (server=${{ matrix.openssh-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-sftp) }}

    uses: ./.github/workflows/test-sftp.yml
    with:
      openssh-version: ${{ matrix.openssh-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  tests-samba:
    name: Run Samba tests (server=${{ matrix.server-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-samba) }}

    uses: ./.github/workflows/test-samba.yml
    with:
      server-version: ${{ matrix.server-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}


  tests-webdav:
    name: Run WebDAV tests (server=${{ matrix.webdav-version }}, pydantic=${{ matrix.pydantic-version }}, python=${{ matrix.python-version }}, os=${{ matrix.os }})
    needs: [get-matrix]
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.get-matrix.outputs.matrix-webdav) }}

    uses: ./.github/workflows/test-webdav.yml
    with:
      webdav-version: ${{ matrix.webdav-version }}
      pydantic-version: ${{ matrix.pydantic-version }}
      python-version: ${{ matrix.python-version }}
      os: ${{ matrix.os }}

  all_done:
    name: Tests done
    runs-on: ubuntu-latest
    needs:
      - tests-core
      - tests-clickhouse
      - tests-hive
      - tests-kafka
      - tests-local-fs
      - tests-mongodb
      - tests-mssql
      - tests-mysql
      - tests-oracle
      - tests-postgres
      - tests-teradata
      - tests-ftp
      - tests-ftps
      - tests-hdfs
      - tests-s3
      - tests-sftp
      - tests-samba
      - tests-webdav

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}

      - name: Cache pip
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-coverage

      - name: Upgrade pip
        run: python -m pip install --upgrade pip setuptools wheel

      - name: Install dependencies
        run: pip install -I coverage pytest

      - name: Download all coverage reports
        uses: actions/download-artifact@v4
        with:
          path: reports/
          pattern: coverage-*
          merge-multiple: true

      - name: Generate coverate reports
        run: ./combine_coverage.sh

      - name: Check coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./reports/coverage.xml
          fail_ci_if_error: true
          plugin: noop

      - name: All done
        run: echo 1