-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Test/add onlineddl scheduler test (#337)
* test: add onlineddl scheduler test Signed-off-by: <919179287@qq.com> * test: alter ddl_strategy to mysql Signed-off-by: <919179287@qq.com> * fix: typo Signed-off-by: <919179287@qq.com> * test: remove testcast which about vitess mode Signed-off-by: <919179287@qq.com> * test: add onlineDDL e2e test Signed-off-by: <919179287@qq.com> * test: move onlineddl scheduler yaml into workflow Signed-off-by: <919179287@qq.com> * test: rm e2e log Signed-off-by: <919179287@qq.com> * test: rename onlineDDL e2e test Signed-off-by: <919179287@qq.com> * test: rename onlineDDL scheduler group name Signed-off-by: <919179287@qq.com> * test: format onlineDDL scheduler yaml Signed-off-by: <919179287@qq.com> * test: rename scheduler yaml name to OnlineDDL scheduler Signed-off-by: <919179287@qq.com> * test: modify ddlstrategy from mysql to vitess Signed-off-by: <919179287@qq.com> * test: add more onlineddl_scheduler_test Signed-off-by: <919179287@qq.com> * test: remove Idempotent submission e2e testcase Signed-off-by: <919179287@qq.com> * test: add onlineddl_vrepl_mysql57 test Signed-off-by: <919179287@qq.com> * test: add Check test results Signed-off-by: <919179287@qq.com> * test: add start e2e test sh file Signed-off-by: <919179287@qq.com> * test: remove online ddl testcase via vtctl Signed-off-by: <919179287@qq.com> * test: modify shard to default '0'; Signed-off-by: <919179287@qq.com> * test: fix vrepl e2e testcase Signed-off-by: <919179287@qq.com> * fix: add onlineddl testcase for mysql57 and mysql80 Signed-off-by: earayu <earayu@163.com> --------- Signed-off-by: <919179287@qq.com> Signed-off-by: earayu <earayu@163.com> Co-authored-by: earayu <earayu@163.com>
- Loading branch information
Showing
11 changed files
with
889 additions
and
461 deletions.
There are no files selected for viewing
140 changes: 140 additions & 0 deletions
140
.github/workflows/cluster_endtoend_onlineddl_scheduler.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,140 @@ | ||
# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" | ||
|
||
name: Cluster (onlineddl_scheduler) | ||
on: | ||
pull_request: | ||
workflow_dispatch: | ||
push: | ||
branches: | ||
- main | ||
concurrency: | ||
group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler)') | ||
cancel-in-progress: true | ||
|
||
env: | ||
GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" | ||
|
||
jobs: | ||
build: | ||
name: Run endtoend OnlineDDL tests on Cluster | ||
runs-on: ubuntu-22.04 | ||
|
||
steps: | ||
- name: Skip CI | ||
run: | | ||
if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then | ||
echo "skipping CI due to the 'Skip CI' label" | ||
exit 1 | ||
fi | ||
- name: Check if workflow needs to be skipped | ||
id: skip-workflow | ||
run: | | ||
skip='false' | ||
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then | ||
skip='true' | ||
fi | ||
echo Skip ${skip} | ||
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT | ||
- name: Check out code | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' | ||
uses: actions/checkout@v3 | ||
|
||
- name: Check for changes in relevant files | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' | ||
uses: frouioui/paths-filter@main | ||
id: changes | ||
with: | ||
token: '' | ||
filters: | | ||
end_to_end: | ||
- 'go/**/*.go' | ||
- 'test.go' | ||
- 'Makefile' | ||
- 'build.env' | ||
- 'go.sum' | ||
- 'go.mod' | ||
- 'proto/*.proto' | ||
- 'tools/**' | ||
- 'config/**' | ||
- 'bootstrap.sh' | ||
- '.github/workflows/cluster_endtoend_onlineddl_scheduler.yml' | ||
- 'go/test/endtoend/onlineddl/vrepl_suite/testdata' | ||
- name: Set up Go | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
uses: actions/setup-go@v3 | ||
with: | ||
go-version: 1.20.1 | ||
|
||
- name: Set up python | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
uses: actions/setup-python@v4 | ||
|
||
- name: Tune the OS | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
run: | | ||
# Limit local port range to not use ports that overlap with server side | ||
# ports that we listen on. | ||
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" | ||
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio | ||
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf | ||
sudo sysctl -p /etc/sysctl.conf | ||
- name: Get dependencies | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
run: | | ||
# Get key to latest MySQL repo | ||
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 | ||
# Setup MySQL 8.0 | ||
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.24-1_all.deb | ||
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections | ||
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* | ||
sudo apt-get update | ||
# Install everything else we need, and configure | ||
sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 | ||
sudo service mysql stop | ||
sudo service etcd stop | ||
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ | ||
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | ||
go mod download | ||
# install JUnit report formatter | ||
go install github.com/vitessio/go-junit-report@HEAD | ||
- name: Run cluster endtoend test | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
timeout-minutes: 45 | ||
run: | | ||
# We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file | ||
# which musn't be more than 107 characters long. | ||
export VTDATAROOT="/tmp/" | ||
source build.env | ||
set -x | ||
# run the tests however you normally do, then produce a JUnit XML file | ||
eatmydata -- go run test.go -docker=false -follow -shard onlineddl_scheduler | tee -a output.txt | go-junit-report -set-exit-code > report.xml | ||
- name: Print test output and Record test result | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() | ||
run: | | ||
# print test output | ||
cat output.txt | ||
- name: Check test results | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() | ||
run: | | ||
# check if any test failed | ||
if grep -q -- '--- FAIL' output.txt; then | ||
echo "Testcase is failed" | ||
exit 1 | ||
elif grep -q -- 'FAIL vitess.io' output.txt; then | ||
echo "Testcase is failed" | ||
exit 1 | ||
else | ||
echo "Testcase is successful" | ||
fi |
151 changes: 151 additions & 0 deletions
151
.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,151 @@ | ||
# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" | ||
|
||
name: Cluster (onlineddl_scheduler) mysql57 | ||
on: | ||
# pull_request: | ||
workflow_dispatch: | ||
push: | ||
branches: | ||
- main | ||
concurrency: | ||
group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql57') | ||
cancel-in-progress: true | ||
|
||
env: | ||
GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" | ||
|
||
jobs: | ||
build: | ||
name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql57 | ||
runs-on: ubuntu-22.04 | ||
|
||
steps: | ||
- name: Skip CI | ||
run: | | ||
if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then | ||
echo "skipping CI due to the 'Skip CI' label" | ||
exit 1 | ||
fi | ||
- name: Check if workflow needs to be skipped | ||
id: skip-workflow | ||
run: | | ||
skip='false' | ||
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then | ||
skip='true' | ||
fi | ||
echo Skip ${skip} | ||
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT | ||
- name: Check out code | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' | ||
uses: actions/checkout@v3 | ||
|
||
- name: Check for changes in relevant files | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' | ||
uses: frouioui/paths-filter@main | ||
id: changes | ||
with: | ||
token: '' | ||
filters: | | ||
end_to_end: | ||
- 'go/**/*.go' | ||
- 'test.go' | ||
- 'Makefile' | ||
- 'build.env' | ||
- 'go.sum' | ||
- 'go.mod' | ||
- 'proto/*.proto' | ||
- 'tools/**' | ||
- 'config/**' | ||
- 'bootstrap.sh' | ||
- '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml' | ||
- 'go/test/endtoend/onlineddl/vrepl_suite/testdata' | ||
- name: Set up Go | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
uses: actions/setup-go@v3 | ||
with: | ||
go-version: 1.20.1 | ||
|
||
- name: Set up python | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
uses: actions/setup-python@v4 | ||
|
||
- name: Tune the OS | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
run: | | ||
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" | ||
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio | ||
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf | ||
sudo sysctl -p /etc/sysctl.conf | ||
- name: Get dependencies | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
run: | | ||
sudo apt-get update | ||
# Uninstall any previously installed MySQL first | ||
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ | ||
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld | ||
sudo systemctl stop apparmor | ||
sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common | ||
sudo apt-get -y autoremove | ||
sudo apt-get -y autoclean | ||
sudo deluser mysql | ||
sudo rm -rf /var/lib/mysql | ||
sudo rm -rf /etc/mysql | ||
# Get key to latest MySQL repo | ||
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 | ||
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.24-1_all.deb | ||
# Bionic packages are still compatible for Jammy since there's no MySQL 5.7 | ||
# packages for Jammy. | ||
echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections | ||
echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections | ||
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* | ||
sudo apt-get update | ||
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses5 | ||
sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata | ||
sudo service mysql stop | ||
sudo service etcd stop | ||
# install JUnit report formatter | ||
go install github.com/vitessio/go-junit-report@HEAD | ||
- name: Run cluster endtoend test | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | ||
timeout-minutes: 45 | ||
run: | | ||
# We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file | ||
# which musn't be more than 107 characters long. | ||
export VTDATAROOT="/tmp/" | ||
source build.env | ||
set -x | ||
# run the tests however you normally do, then produce a JUnit XML file | ||
eatmydata -- go run test.go -docker=false -follow -shard onlineddl_scheduler | tee -a output.txt | go-junit-report -set-exit-code > report.xml | ||
- name: Print test output and Record test result | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() | ||
run: | | ||
# print test output | ||
cat output.txt | ||
- name: Check test results | ||
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() | ||
run: | | ||
# check if any test failed | ||
if grep -q -- '--- FAIL' output.txt; then | ||
echo "Testcase is failed" | ||
exit 1 | ||
elif grep -q -- 'FAIL vitess.io' output.txt; then | ||
echo "Testcase is failed" | ||
exit 1 | ||
else | ||
echo "Testcase is successful" | ||
fi |
Oops, something went wrong.