Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[tests][ci] Refactor ci #3626

Merged
merged 2 commits into from
Oct 15, 2024
Merged

[tests][ci] Refactor ci #3626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
58ca2fc
[hotfix][test] Reorganize test cases
yuxiqian Oct 11, 2024
700eced
[hotfix][ci] Optimize CI performance by implementing parallel execution
GOODBOY008 Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
349 changes: 32 additions & 317 deletions .github/workflows/flink_cdc.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,78 +30,13 @@ on:
- 'docs/**'
- 'README.md'

# Concurrency strategy:
# github.workflow: distinguish this workflow from others
# github.event_name: distinguish `push` event from `pull_request` event
# github.event.number: set to the number of the pull request if `pull_request` event
# github.run_id: otherwise, it's a `push` or `schedule` event, only cancel if we rerun the workflow
#
# Reference:
# https://docs.github.com/en/actions/using-jobs/using-concurrency
# https://docs.github.com/en/actions/learn-github-actions/contexts#github-context
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true

env:
MODULES_CORE: "\
flink-cdc-cli,\
flink-cdc-common,\
flink-cdc-composer,\
flink-cdc-runtime"

MODULES_PIPELINE_CONNECTORS: "\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-values,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-mysql,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-doris,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-starrocks,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-kafka,\
flink-cdc-connect/flink-cdc-pipeline-connectors/flink-cdc-pipeline-connector-paimon"

MODULES_MYSQL: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mysql-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mysql-cdc"

MODULES_POSTGRES: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-postgres-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-postgres-cdc"

MODULES_ORACLE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oracle-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oracle-cdc"

MODULES_MONGODB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-mongodb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-mongodb-cdc"

MODULES_SQLSERVER: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-sqlserver-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-sqlserver-cdc"

MODULES_TIDB: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-tidb-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-tidb-cdc"

MODULES_OCEANBASE: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-oceanbase-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-oceanbase-cdc"

MODULES_DB2: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-db2-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-db2-cdc"

MODULES_VITESS: "\
flink-cdc-connect/flink-cdc-source-connectors/flink-connector-vitess-cdc,\
flink-cdc-connect/flink-cdc-source-connectors/flink-sql-connector-vitess-cdc"

MODULES_PIPELINE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-pipeline-e2e-tests"

MODULES_SOURCE_E2E: "\
flink-cdc-e2e-tests/flink-cdc-source-e2e-tests"

jobs:
license_check:
name: License Check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
Expand All @@ -116,260 +51,40 @@ jobs:
run: mvn --no-snapshot-updates -B package -DskipTests
- name: Run license check
run: gem install rubyzip -v 2.3.0 && ./tools/ci/license_check.rb

compile_and_test:
needs: license_check
# Only run the CI pipeline for the flink-cdc-connectors repository
# if: github.repository == 'apache/flink-cdc-connectors'
runs-on: ubuntu-latest
ut:
strategy:
matrix:
jdk: [ 8 ]
module: [ "core",
"pipeline_connectors",
"mysql",
"postgres",
"oracle",
"mongodb",
"sqlserver",
"tidb",
"oceanbase",
"db2",
"vitess",
"pipeline_e2e",
"source_e2e"
]
timeout-minutes: 120
env:
MVN_COMMON_OPTIONS: -Dmaven.wagon.http.pool=false \
-Dorg.slf4j.simpleLogger.showDateTime=true \
-Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss.SSS \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
--no-snapshot-updates -B \
--settings /home/vsts/work/1/s/tools/ci/google-mirror-settings.xml \
-Dfast -Dlog.dir=/home/vsts/work/_temp/debug_files \
-Dlog4j.configurationFile=file:///home/vsts/work/1/s/tools/ci/log4j.properties
steps:
- run: echo "Running CI pipeline for JDK version ${{ matrix.jdk }}"

- name: Clean up disk space
run: |
set -euo pipefail

echo "Disk space before cleanup"
df -h

echo "Cleaning up disk space"
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo docker image prune --all --force

echo "Disk space after cleanup"
df -h

- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true

- name: Set JDK
uses: actions/setup-java@v4
with:
java-version: ${{ matrix.jdk }}
distribution: 'temurin'
cache: 'maven'

- name: Set Maven 3.8.6
uses: stCarolas/setup-maven@v5
with:
maven-version: 3.8.6

- name: Compile and test ${{ matrix.module }}
timeout-minutes: 90
run: |
set -o pipefail

case ${{ matrix.module }} in
("core")
modules=${{ env.MODULES_CORE }}
;;
("pipeline_connectors")
modules=${{ env.MODULES_PIPELINE_CONNECTORS }}
;;
("mysql")
modules=${{ env.MODULES_MYSQL }}
;;
("postgres")
modules=${{ env.MODULES_POSTGRES }}
;;
("oracle")
modules=${{ env.MODULES_ORACLE }}
;;
("mongodb")
modules=${{ env.MODULES_MONGODB }}
;;
("sqlserver")
modules=${{ env.MODULES_SQLSERVER }}
;;
("tidb")
modules=${{ env.MODULES_TIDB }}
;;
("oceanbase")
modules=${{ env.MODULES_OCEANBASE }}
;;
("db2")
modules=${{ env.MODULES_DB2 }}
;;
("vitess")
modules=${{ env.MODULES_VITESS }}
;;
("pipeline_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_PIPELINE_E2E }}"
modules=${{ env.MODULES_PIPELINE_E2E }}
;;
("source_e2e")
compile_modules="${{ env.MODULES_CORE }},${{ env.MODULES_PIPELINE_CONNECTORS }},${{ env.MODULES_MYSQL }},${{ env.MODULES_POSTGRES }},${{ env.MODULES_ORACLE }},${{ env.MODULES_MONGODB }},${{ env.MODULES_SQLSERVER }},${{ env.MODULES_TIDB }},${{ env.MODULES_OCEANBASE }},${{ env.MODULES_DB2 }},${{ env.MODULES_VITESS }},${{ env.MODULES_SOURCE_E2E }}"
modules=${{ env.MODULES_SOURCE_E2E }}
;;
esac

if [ ${{ matrix.module }} != "pipeline_e2e" ] && [ ${{ matrix.module }} != "source_e2e" ]; then
compile_modules=$modules
fi

mvn --no-snapshot-updates -B -DskipTests -pl $compile_modules -am install && mvn --no-snapshot-updates -B -pl $modules verify

- name: Print JVM thread dumps when cancelled
if: ${{ failure() }}
run: |
# ----------------------------------------------------------------------------
# Copyright 2023 The Netty Project
#
# ----------------------------------------------------------------------------
# Source: https://github.com/netty/netty/blob/main/.github/actions/thread-dump-jvms/action.yml
echo "$OSTYPE"
if [[ "$OSTYPE" == "linux-gnu"* ]] && command -v sudo &> /dev/null; then
echo "Setting up JVM thread dumps"
# use jattach so that Java processes in docker containers are also covered
# download jattach
curl -s -L -o /tmp/jattach https://github.com/apangin/jattach/releases/download/v2.1/jattach
if command -v sha256sum &> /dev/null; then
# verify hash of jattach binary
sha256sum -c <(echo "07885fdc782e02e7302c6d190f54c3930afa10a38140365adf54076ec1086a8e /tmp/jattach") || exit 1
fi
chmod +x /tmp/jattach
for java_pid in $(sudo pgrep java); do
echo "----------------------- pid $java_pid -----------------------"
echo "command line: $(sudo cat /proc/$java_pid/cmdline | xargs -0 echo)"
sudo /tmp/jattach $java_pid jcmd VM.command_line || true
sudo /tmp/jattach $java_pid jcmd "Thread.print -l"
sudo /tmp/jattach $java_pid jcmd GC.heap_info || true
done
else
for java_pid in $(jps -q -J-XX:+PerfDisableSharedMem); do
echo "----------------------- pid $java_pid -----------------------"
jcmd $java_pid VM.command_line || true
jcmd $java_pid Thread.print -l
jcmd $java_pid GC.heap_info || true
done
fi
exit 0


migration_test_ut:
needs: license_check
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
submodules: true
- name: Compile snapshot CDC version
run: mvn --no-snapshot-updates -B install -DskipTests
- name: Run migration tests
run: cd flink-cdc-migration-tests && mvn clean verify

pipeline_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
java-version: [ '8' ]
module: [ 'core', 'pipeline_connectors', 'mysql', 'postgres', 'oracle', 'mongodb6', 'mongodb7', 'sqlserver', 'tidb', 'oceanbase', 'db2', 'vitess' ]
name: Unit Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
module: ${{ matrix.module }}
pipeline_e2e:
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: FLINK_HOME=./flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test

data_stream_migration_test:
needs: migration_test_ut
runs-on: ubuntu-latest
java-version: [ '8' ]
flink-version: ['1.17.2', '1.18.1', '1.19.1', '1.20.0']
module: [ 'pipeline_e2e' ]
name: Pipeline E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
flink-version: ${{ matrix.flink-version }}
module: ${{ matrix.module }}
source_e2e:
strategy:
matrix:
java-version: [ '8', '11' ]

steps:
- uses: actions/checkout@v4
- name: Set up Ruby
uses: ruby/setup-ruby@v1
with:
ruby-version: 3.0
bundler-cache: true # runs 'bundle install' and caches installed gems automatically
- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: temurin
cache: maven
- name: Install dependencies
run: gem install terminal-table
- name: Prepare CDC versions
run: CDC_SOURCE_HOME=$PWD ruby tools/mig-test/prepare_libs.rb
- name: Prepare Flink distro
run: wget https://dlcdn.apache.org/flink/flink-1.18.1/flink-1.18.1-bin-scala_2.12.tgz && tar -xzvf flink-1.18.1-bin-scala_2.12.tgz
working-directory: ./tools/mig-test
- name: Patch Flink configs
run: FLINK_HOME=./flink-1.18.1/ ruby misc/patch_flink_conf.rb
working-directory: ./tools/mig-test
- name: Compile Dummy DataStream Jobs
run: cd datastream && ruby compile_jobs.rb
working-directory: ./tools/mig-test
- name: Start containers
run: cd conf && docker compose up -d
working-directory: ./tools/mig-test
- name: Run migration tests
run: cd datastream && FLINK_HOME=../flink-1.18.1/ ruby run_migration_test.rb
working-directory: ./tools/mig-test
- name: Stop containers
if: always()
run: cd conf && docker compose down
working-directory: ./tools/mig-test
java-version: [ '8' ]
flink-version: ['1.16.3', '1.17.2', '1.18.1', '1.19.1', '1.20.0']
module: [ 'source_e2e' ]
name: Source E2E Tests
uses: ./.github/workflows/flink_cdc_base.yml
with:
java-version: ${{ matrix.java-version }}
flink-version: ${{ matrix.flink-version }}
module: ${{ matrix.module }}
migration_test:
name: Migration Tests
uses: ./.github/workflows/flink_cdc_migration_test.yml
Loading
Loading