From 8e37a7f2e3e533f830f4fe0d85f6114d46550d51 Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Wed, 14 Jun 2023 11:41:41 +0300 Subject: [PATCH 1/2] chore: split unit tests --- .github/workflows/tests.yaml | 122 ++++++++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 607c45b390..6bd9c213da 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -119,13 +119,114 @@ jobs: TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST }} TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME }} TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }} - run: make test exclude=/rudder-server/warehouse + run: make test exclude=/rudder-server/(jobsdb|integration_test|processor|regulation-worker|router|suppression-backup-service|warehouse) - name: Upload coverage report uses: actions/upload-artifact@v3 with: name: unit path: coverage.txt - + jobsdb-unit: + name: JobsDB Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./jobsdb/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: jobsdb-unit + path: coverage.txt + integration-unit: + name: Integration Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./integration_test/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: integration-unit + path: coverage.txt + processor-unit: + name: Processor Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./processor/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: processor-unit + path: coverage.txt + regulation-worker-unit: + name: Regulation Worker Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./regulation-worker/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: regulation-worker-unit + path: coverage.txt + router-unit: + name: Router Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./router/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: router-unit + path: coverage.txt + suppression-backup-service-unit: + name: Suppression Backup Service Unit + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '~1.20.3' + check-latest: true + - run: go version + - run: go mod download + - run: make test package=./suppression-backup-service/... + - name: Upload coverage report + uses: actions/upload-artifact@v3 + with: + name: suppression-backup-service-unit + path: coverage.txt warehouse-unit: name: Warehouse Unit runs-on: 'ubuntu-20.04' @@ -148,8 +249,17 @@ jobs: path: coverage.txt coverage: - needs: [warehouse-integration, unit, warehouse-unit] runs-on: 'ubuntu-20.04' + needs: + - warehouse-integration + - unit + - jobsdb-unit + - integration-unit + - processor-unit + - regulation-worker-unit + - router-unit + - suppression-backup-service-unit + - warehouse-unit steps: - name: Checkout uses: actions/checkout@v3 @@ -176,6 +286,12 @@ jobs: - integration - warehouse-integration - unit + - jobsdb-unit + - integration-unit + - processor-unit + - regulation-worker-unit + - router-unit + - suppression-backup-service-unit - warehouse-unit steps: - uses: re-actors/alls-green@v1.2.2 From 5bfa8a2e4e196dcfd4698603022a874dd7a8daca Mon Sep 17 00:00:00 2001 From: Aris Tzoumas Date: Wed, 14 Jun 2023 12:15:24 +0300 Subject: [PATCH 2/2] fixup! chore: split unit tests --- .github/workflows/tests.yaml | 165 ++++++----------------------------- 1 file changed, 26 insertions(+), 139 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6bd9c213da..04fd31d2fd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -8,7 +8,7 @@ on: pull_request: jobs: integration: - name: Integrations + name: Integration runs-on: 'ubuntu-20.04' strategy: matrix: @@ -23,20 +23,19 @@ jobs: - run: go version - run: go mod download # Not required, used to segregate module download vs test times - - name: Integration test for enterprise + - name: enterprise if: matrix.FEATURES == 'enterprise' run: go test -v ./integration_test/docker_test/docker_test.go -count 1 env: ENTERPRISE_TOKEN: ${{ secrets.ENTERPRISE_TOKEN }} - - name: Integration test for oss + - name: oss if: matrix.FEATURES == 'oss' run: go test -v ./integration_test/docker_test/docker_test.go -count 1 env: RSERVER_ENABLE_MULTITENANCY: ${{ matrix.MULTITENANCY }} - warehouse-integration: - name: Warehouse Service Integration + name: Warehouse Integration runs-on: 'ubuntu-20.04' strategy: fail-fast: false @@ -99,7 +98,6 @@ jobs: with: name: ${{ matrix.destination }} path: coverage.txt - unit: name: Unit runs-on: 'ubuntu-20.04' @@ -109,127 +107,21 @@ jobs: with: go-version: '~1.20.4' check-latest: true - - run: go version - run: go mod download # Not required, used to segregate module download vs test times - - env: - TEST_KAFKA_CONFLUENT_CLOUD_HOST: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_HOST }} - TEST_KAFKA_CONFLUENT_CLOUD_KEY: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_KEY }} - TEST_KAFKA_CONFLUENT_CLOUD_SECRET: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_SECRET }} - TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST }} - TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME }} - TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }} - run: make test exclude=/rudder-server/(jobsdb|integration_test|processor|regulation-worker|router|suppression-backup-service|warehouse) + - run: make test exclude="/rudder-server/(jobsdb|integration_test|processor|regulation-worker|router|services|suppression-backup-service|warehouse)" - name: Upload coverage report uses: actions/upload-artifact@v3 with: name: unit path: coverage.txt - jobsdb-unit: - name: JobsDB Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./jobsdb/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: jobsdb-unit - path: coverage.txt - integration-unit: - name: Integration Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./integration_test/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: integration-unit - path: coverage.txt - processor-unit: - name: Processor Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./processor/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: processor-unit - path: coverage.txt - regulation-worker-unit: - name: Regulation Worker Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./regulation-worker/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: regulation-worker-unit - path: coverage.txt - router-unit: - name: Router Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./router/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: router-unit - path: coverage.txt - suppression-backup-service-unit: - name: Suppression Backup Service Unit - runs-on: 'ubuntu-20.04' - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v4 - with: - go-version: '~1.20.3' - check-latest: true - - run: go version - - run: go mod download - - run: make test package=./suppression-backup-service/... - - name: Upload coverage report - uses: actions/upload-artifact@v3 - with: - name: suppression-backup-service-unit - path: coverage.txt - warehouse-unit: - name: Warehouse Unit + package-unit: + name: Package Unit runs-on: 'ubuntu-20.04' + strategy: + fail-fast: false + matrix: + package: [ jobsdb, integration_test, processor, regulation-worker, router, services, suppression-backup-service, warehouse ] steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 @@ -238,28 +130,29 @@ jobs: check-latest: true - run: go version - run: go mod download - - env: + - name: Package Unit [ ${{ matrix.package }} ] + env: + TEST_KAFKA_CONFLUENT_CLOUD_HOST: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_HOST }} + TEST_KAFKA_CONFLUENT_CLOUD_KEY: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_KEY }} + TEST_KAFKA_CONFLUENT_CLOUD_SECRET: ${{ secrets.TEST_KAFKA_CONFLUENT_CLOUD_SECRET }} + TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_HOST }} + TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_EVENTHUB_NAME }} + TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING: ${{ secrets.TEST_KAFKA_AZURE_EVENT_HUBS_CLOUD_CONNECTION_STRING }} TEST_S3_DATALAKE_CREDENTIALS: ${{ secrets.TEST_S3_DATALAKE_CREDENTIALS }} BIGQUERY_INTEGRATION_TEST_CREDENTIALS: ${{ secrets.BIGQUERY_INTEGRATION_TEST_CREDENTIALS }} - run: make test package=./warehouse/... + run: make test package=./${{ matrix.package }}/... - name: Upload coverage report uses: actions/upload-artifact@v3 with: - name: warehouse-unit + name: ${{ matrix.package }}-unit path: coverage.txt - coverage: + name: Coverage runs-on: 'ubuntu-20.04' needs: - warehouse-integration - unit - - jobsdb-unit - - integration-unit - - processor-unit - - regulation-worker-unit - - router-unit - - suppression-backup-service-unit - - warehouse-unit + - package-unit steps: - name: Checkout uses: actions/checkout@v3 @@ -278,21 +171,15 @@ jobs: with: fail_ci_if_error: true files: ./coverage.txt - - all-tests-successful: + all-green: + name: All if: always() runs-on: ubuntu-latest needs: - integration - warehouse-integration - unit - - jobsdb-unit - - integration-unit - - processor-unit - - regulation-worker-unit - - router-unit - - suppression-backup-service-unit - - warehouse-unit + - package-unit steps: - uses: re-actors/alls-green@v1.2.2 with: