diff --git a/.github/workflows/new_endtoend_test.yml b/.github/workflows/new_endtoend_test.yml new file mode 100644 index 0000000000..120976fb92 --- /dev/null +++ b/.github/workflows/new_endtoend_test.yml @@ -0,0 +1,118 @@ +name: "New EndToEnd Test" + +on: + pull_request: + types: [opened, synchronize, reopened] + push: + branches: + - main + - 'release-*' + tags: + - '*' + workflow_dispatch: + +env: + IMAGE_NAME: wescale_ci_image + REGISTRY: ghcr.io + IMAGE_TAG: test-${{ github.sha }} + MYSQL_VERSION: 8.0.32 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-endtoend + cancel-in-progress: true + +jobs: + build-image: + permissions: + contents: read + packages: write + uses: ./.github/workflows/build_image.yml + with: + branch: ${{ github.ref }} + image_name: ${{ github.repository_owner }}/wescale_ci_image + tags: test-${{ github.sha }} + platforms: linux/amd64 + want_push: false + want_load: true + want_artifact: true + artifact_name: 'image.tar' + + setup: + name: "New EndToEnd Test" + needs: build-image + runs-on: ubuntu-latest + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Checkout code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Download Docker image + uses: actions/download-artifact@v3 + with: + name: image.tar + path: /tmp + + - name: Load Docker image + run: | + docker load < /tmp/image.tar + echo "Verifying image loaded:" + docker images + + - name: Set up cluster + run: | + MYSQL_IMG="mysql/mysql-server:${{ env.MYSQL_VERSION }}" + WESCALE_CI_IMAGE="${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}" + + docker network create wescale-network + + docker run -itd --network wescale-network --name mysql-server \ + -p 3306:3306 \ + -e MYSQL_ROOT_PASSWORD=passwd \ + -e MYSQL_ROOT_HOST=% \ + -e MYSQL_LOG_CONSOLE=true \ + $MYSQL_IMG \ + --bind-address=0.0.0.0 \ + --port=3306 \ + --log-bin=binlog \ + --gtid_mode=ON \ + --enforce_gtid_consistency=ON \ + --log_replica_updates=ON \ + --binlog_format=ROW + + docker run -itd --network wescale-network --name wescale \ + -p 15306:15306 \ + -w /vt/examples/wesql-server \ + -e MYSQL_ROOT_USER=root \ + -e MYSQL_ROOT_PASSWORD=passwd \ + -e MYSQL_PORT=3306 \ + -e MYSQL_HOST=mysql-server \ + -e CONFIG_PATH=/vt/config/wescale/default \ + $WESCALE_CI_IMAGE \ + /vt/examples/wesql-server/init_single_node_cluster.sh + + - name: Run EndToEnd test + run: | + cd endtoend + go test ./... -v + + - name: Print Wescale logs + run: | + docker logs wescale \ No newline at end of file diff --git a/examples/wesql-server/conf/vtgate.cnf b/config/wescale/default/vtgate.cnf similarity index 100% rename from examples/wesql-server/conf/vtgate.cnf rename to config/wescale/default/vtgate.cnf diff --git a/examples/wesql-server/conf/vttablet.cnf b/config/wescale/default/vttablet.cnf similarity index 100% rename from examples/wesql-server/conf/vttablet.cnf rename to config/wescale/default/vttablet.cnf diff --git a/docker/wesqlscale/Dockerfile.release b/docker/wesqlscale/Dockerfile.release index 2d676f4e48..6eaa30b4c1 100644 --- a/docker/wesqlscale/Dockerfile.release +++ b/docker/wesqlscale/Dockerfile.release @@ -95,6 +95,7 @@ RUN cp /vt/src/vitess/bin/vtctld /vt/bin/ && \ cp /vt/src/vitess/bin/protoc /vt/bin && \ cp /vt/src/vitess/bin/etcd /vt/bin && \ cp /vt/src/vitess/bin/etcdctl /vt/bin && \ + cp -rf $VTSOURCEROOT/config /vt && \ cp -rf $VTSOURCEROOT/examples /vt && \ rm -rf /vt/src diff --git a/endtoend/framework/clusters/single_node_cluster.go b/endtoend/framework/clusters/single_node_cluster.go new file mode 100644 index 0000000000..6e31699600 --- /dev/null +++ b/endtoend/framework/clusters/single_node_cluster.go @@ -0,0 +1,140 @@ +package clusters + +import ( + "database/sql" + "flag" + "fmt" + "github.com/wesql/wescale/endtoend/framework" +) + +func (s *SingleNodeCluster) RegisterFlagsForSingleNodeCluster() { + flag.StringVar(&s.mysqlHost, fmt.Sprintf("%s_mysqlHost", s.ClusterName), s.mysqlHost, "Host of the MySQL server") + flag.IntVar(&s.mysqlPort, fmt.Sprintf("%s_mysqlPort", s.ClusterName), s.mysqlPort, "Port of the MySQL server") + flag.StringVar(&s.mysqlUser, fmt.Sprintf("%s_mysqlUser", s.ClusterName), s.mysqlUser, "User for the MySQL server") + flag.StringVar(&s.mysqlPasswd, fmt.Sprintf("%s_mysqlPasswd", s.ClusterName), s.mysqlPasswd, "Password for the MySQL server") + + flag.StringVar(&s.wescaleHost, fmt.Sprintf("%s_wescaleHost", s.ClusterName), s.wescaleHost, "Host of the WeScale server") + flag.IntVar(&s.wescalePort, fmt.Sprintf("%s_wescalePort", s.ClusterName), s.wescalePort, "Port of the WeScale server") + flag.StringVar(&s.wescaleUser, fmt.Sprintf("%s_wescaleUser", s.ClusterName), s.wescaleUser, "User for the WeScale server") + flag.StringVar(&s.wescalePasswd, fmt.Sprintf("%s_wescalePasswd", s.ClusterName), s.wescalePasswd, "Password for the WeScale server") +} + +type SingleNodeCluster struct { + ClusterName string + + mysqlHost string + mysqlPort int + mysqlUser string + mysqlPasswd string + + wescaleHost string + wescalePort int + wescaleUser string + wescalePasswd string + + DbName string + SetUpScript string + CleanupScript string + + MysqlDb *sql.DB + WescaleDb *sql.DB +} + +func NewDefaultSingleNodeCluster() *SingleNodeCluster { + return newCustomSingleNodeCluster( + "default", + "127.0.0.1", + 3306, + "root", + "passwd", + "127.0.0.1", + 15306, + "root", + "passwd", + ) +} + +func newCustomSingleNodeCluster(clusterName string, + mysqlHost string, mysqlPort int, mysqlUser string, mysqlPasswd string, + wescaleHost string, wescalePort int, wescaleUser string, wescalePasswd string) *SingleNodeCluster { + s := &SingleNodeCluster{ + ClusterName: clusterName, + mysqlHost: mysqlHost, + mysqlPort: mysqlPort, + mysqlUser: mysqlUser, + mysqlPasswd: mysqlPasswd, + wescaleHost: wescaleHost, + wescalePort: wescalePort, + wescaleUser: wescaleUser, + wescalePasswd: wescalePasswd, + } + return s +} + +// SetUpSingleNodeCluster creates a single node cluster with a MySQL and WeScale database. +// dbName can be an empty string if no database is needed. +// setupScript can be an empty string if no setup script is needed. +// cleanupScript can be an empty string if no cleanup script is needed. +func (s *SingleNodeCluster) SetUp(dbName string, setupScript string, cleanupScript string) error { + // Create the database + db, err := framework.NewMysqlConnectionPool(s.wescaleHost, s.wescalePort, s.wescaleUser, s.wescalePasswd, "") + if err != nil { + return err + } + defer db.Close() + if dbName != "" { + _, err = db.Exec(fmt.Sprintf("create database if not exists `%s`", dbName)) + if err != nil { + return err + } + } + + // Create the connection pools + mysqlDb, err := framework.NewMysqlConnectionPool(s.mysqlHost, s.mysqlPort, s.mysqlUser, s.mysqlPasswd, dbName) + if err != nil { + return err + } + wescaleDb, err := framework.NewMysqlConnectionPool(s.wescaleHost, s.wescalePort, s.wescaleUser, s.wescalePasswd, dbName) + if err != nil { + return err + } + + // Execute Set Up Script + if setupScript != "" { + err = framework.ExecuteSqlScript(wescaleDb, setupScript) + if err != nil { + return err + } + } + + s.DbName = dbName + s.SetUpScript = setupScript + s.CleanupScript = cleanupScript + s.MysqlDb = mysqlDb + s.WescaleDb = wescaleDb + return nil +} + +func (s *SingleNodeCluster) CleanUp() error { + // Execute Clean Up Script + if s.CleanupScript != "" { + err := framework.ExecuteSqlScript(s.WescaleDb, s.CleanupScript) + if err != nil { + return err + } + } + + if s.MysqlDb != nil { + err := s.MysqlDb.Close() + if err != nil { + return err + } + } + if s.WescaleDb != nil { + err := s.WescaleDb.Close() + if err != nil { + return err + } + } + return nil +} diff --git a/endtoend/framework/connection_utils.go b/endtoend/framework/connection_utils.go new file mode 100644 index 0000000000..051955e501 --- /dev/null +++ b/endtoend/framework/connection_utils.go @@ -0,0 +1,34 @@ +package framework + +import ( + "database/sql" + "fmt" + "github.com/go-sql-driver/mysql" +) + +func newMysqlConfig(host string, port int, user string, passwd string, dbName string) *mysql.Config { + return &mysql.Config{ + Net: "tcp", + Addr: fmt.Sprintf("%s:%d", host, port), + User: user, + Passwd: passwd, + AllowNativePasswords: true, + DBName: dbName, + } +} + +func NewMysqlConnectionPool(host string, port int, user string, passwd string, dbName string) (*sql.DB, error) { + c := newMysqlConfig(host, port, user, passwd, dbName) + c.MultiStatements = true + db, err := sql.Open("mysql", c.FormatDSN()) + if err != nil { + return nil, err + } + + if err := db.Ping(); err != nil { + db.Close() + return nil, err + } + + return db, nil +} diff --git a/endtoend/framework/filter_builder.go b/endtoend/framework/filter_builder.go new file mode 100644 index 0000000000..2d489e97dc --- /dev/null +++ b/endtoend/framework/filter_builder.go @@ -0,0 +1,158 @@ +package framework + +import ( + "bytes" + "fmt" + "text/template" +) + +type FilterBuilder struct { + Name string + Description string + Priority string + Status string + + // Pattern fields + Plans string + FullyQualifiedTables string + QueryRegex string + QueryTemplate string + RequestIPRegex string + UserRegex string + LeadingCommentRegex string + TrailingCommentRegex string + BindVarConds string + + // Execute fields + Action string + ActionArgs string +} + +func NewFilterBuilder(filterName string) *FilterBuilder { + return &FilterBuilder{ + Name: filterName, + Priority: "1000", + Status: "ACTIVE", + } +} + +func NewWasmFilterBuilder(filterName string, wasmName string) *FilterBuilder { + return &FilterBuilder{ + Name: filterName, + Priority: "1000", + Status: "ACTIVE", + Action: "wasm_plugin", + ActionArgs: fmt.Sprintf("wasm_binary_name=\"%v\"", wasmName), + } +} + +func (fb *FilterBuilder) SetName(name string) *FilterBuilder { + fb.Name = name + return fb +} + +func (fb *FilterBuilder) SetDescription(desc string) *FilterBuilder { + fb.Description = desc + return fb +} + +func (fb *FilterBuilder) SetPriority(priority string) *FilterBuilder { + fb.Priority = priority + return fb +} + +func (fb *FilterBuilder) SetStatus(status string) *FilterBuilder { + fb.Status = status + return fb +} + +func (fb *FilterBuilder) SetPlans(plans string) *FilterBuilder { + fb.Plans = plans + return fb +} + +func (fb *FilterBuilder) SetFullyQualifiedTables(tables string) *FilterBuilder { + fb.FullyQualifiedTables = tables + return fb +} + +func (fb *FilterBuilder) SetQueryRegex(regex string) *FilterBuilder { + fb.QueryRegex = regex + return fb +} + +func (fb *FilterBuilder) SetQueryTemplate(template string) *FilterBuilder { + fb.QueryTemplate = template + return fb +} + +func (fb *FilterBuilder) SetRequestIPRegex(regex string) *FilterBuilder { + fb.RequestIPRegex = regex + return fb +} + +func (fb *FilterBuilder) SetUserRegex(regex string) *FilterBuilder { + fb.UserRegex = regex + return fb +} + +func (fb *FilterBuilder) SetLeadingCommentRegex(regex string) *FilterBuilder { + fb.LeadingCommentRegex = regex + return fb +} + +func (fb *FilterBuilder) SetTrailingCommentRegex(regex string) *FilterBuilder { + fb.TrailingCommentRegex = regex + return fb +} + +func (fb *FilterBuilder) SetBindVarConds(conds string) *FilterBuilder { + fb.BindVarConds = conds + return fb +} + +func (fb *FilterBuilder) SetAction(action string) *FilterBuilder { + fb.Action = action + return fb +} + +func (fb *FilterBuilder) SetActionArgs(args string) *FilterBuilder { + fb.ActionArgs = args + return fb +} + +func (fb *FilterBuilder) Build() (string, error) { + const sqlTemplate = `create filter if not exists {{.Name}} ( + desc='{{.Description}}', + priority='{{.Priority}}', + status='{{.Status}}' +) +with_pattern( + plans='{{.Plans}}', + fully_qualified_table_names='{{.FullyQualifiedTables}}', + query_regex='{{.QueryRegex}}', + query_template='{{.QueryTemplate}}', + request_ip_regex='{{.RequestIPRegex}}', + user_regex='{{.UserRegex}}', + leading_comment_regex='{{.LeadingCommentRegex}}', + trailing_comment_regex='{{.TrailingCommentRegex}}', + bind_var_conds='{{.BindVarConds}}' +) +execute( + action='{{.Action}}', + action_args='{{.ActionArgs}}' +);` + + tmpl, err := template.New("sql").Parse(sqlTemplate) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, fb) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/endtoend/framework/filter_builder_test.go b/endtoend/framework/filter_builder_test.go new file mode 100644 index 0000000000..dd54b7b715 --- /dev/null +++ b/endtoend/framework/filter_builder_test.go @@ -0,0 +1,37 @@ +package framework + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFilterBuilder(t *testing.T) { + sql, err := NewFilterBuilder("trailing_comment_reg"). + SetDescription("test description"). + SetPriority("999"). + SetTrailingCommentRegex("h.*w.*"). + SetAction("FAIL"). + Build() + assert.NoError(t, err) + expect := `create filter if not exists trailing_comment_reg ( + desc='test description', + priority='999', + status='ACTIVE' +) +with_pattern( + plans='', + fully_qualified_table_names='', + query_regex='', + query_template='', + request_ip_regex='', + user_regex='', + leading_comment_regex='', + trailing_comment_regex='h.*w.*', + bind_var_conds='' +) +execute( + action='FAIL', + action_args='' +);` + assert.Equal(t, expect, sql) +} diff --git a/endtoend/framework/filter_utils.go b/endtoend/framework/filter_utils.go new file mode 100644 index 0000000000..abd45fe96d --- /dev/null +++ b/endtoend/framework/filter_utils.go @@ -0,0 +1,28 @@ +package framework + +import ( + "database/sql" + "fmt" + "github.com/stretchr/testify/assert" + "testing" +) + +func AssertFilterExists(t *testing.T, filterName string, db *sql.DB) { + t.Helper() + row := QueryNoError(t, db, fmt.Sprintf("show create filter %s", filterName)) + defer row.Close() + assert.True(t, row.Next()) +} + +func CreateFilterIfNotExists(t *testing.T, builder *FilterBuilder, db *sql.DB) { + t.Helper() + sql, err := builder.Build() + assert.NoError(t, err) + + ExecNoError(t, db, sql) +} + +func DropFilter(t *testing.T, filterName string, db *sql.DB) { + t.Helper() + ExecNoError(t, db, fmt.Sprintf("drop filter %s", filterName)) +} diff --git a/endtoend/framework/sql_utils.go b/endtoend/framework/sql_utils.go new file mode 100644 index 0000000000..8e494f611f --- /dev/null +++ b/endtoend/framework/sql_utils.go @@ -0,0 +1,42 @@ +package framework + +import ( + "database/sql" + "github.com/stretchr/testify/assert" + "log" + "testing" +) + +func ExecuteSqlScript(db *sql.DB, sqlScript string) error { + _, err := db.Exec(sqlScript) + return err +} + +func ExecNoError(t *testing.T, db *sql.DB, sql string, args ...any) { + t.Helper() + log.Println(sql) + _, err := db.Exec(sql, args...) + assert.NoError(t, err) +} + +func QueryNoError(t *testing.T, db *sql.DB, sql string, args ...any) *sql.Rows { + t.Helper() + log.Println(sql) + rows, err := db.Query(sql, args...) + assert.NoError(t, err) + return rows +} + +func ExecWithErrorContains(t *testing.T, db *sql.DB, contains string, sql string, args ...any) { + t.Helper() + log.Println(sql) + _, err := db.Exec(sql, args...) + assert.ErrorContains(t, err, contains) +} + +func QueryWithErrorContains(t *testing.T, db *sql.DB, contains string, sql string, args ...any) { + t.Helper() + log.Println(sql) + _, err := db.Query(sql, args...) + assert.ErrorContains(t, err, contains) +} diff --git a/endtoend/framework/wasm_utils.go b/endtoend/framework/wasm_utils.go new file mode 100644 index 0000000000..ab72ad86ad --- /dev/null +++ b/endtoend/framework/wasm_utils.go @@ -0,0 +1,48 @@ +package framework + +import ( + "bytes" + "crypto/md5" + "database/sql" + "fmt" + "github.com/dsnet/compress/bzip2" + "github.com/stretchr/testify/assert" + "testing" +) + +const wasmRuntime = "wazero" +const wasmCompressAlgorithm = "bzip2" + +func InstallWasm(t *testing.T, wasmBytes []byte, wasmName string, db *sql.DB) { + t.Helper() + hash := calcMd5String32(wasmBytes) + compressedWasmBytes, err := compressByBZip2(wasmBytes) + assert.NoError(t, err) + ExecNoError(t, db, `insert ignore into mysql.wasm_binary(name,runtime,data,compress_algorithm,hash_before_compress) values (?,?,?,?,?)`, + wasmName, wasmRuntime, compressedWasmBytes, wasmCompressAlgorithm, hash) +} + +func UninstallWasm(t *testing.T, wasmName string, db *sql.DB) { + t.Helper() + ExecNoError(t, db, `delete from mysql.wasm_binary where name = ?`, wasmName) +} + +func calcMd5String32(data []byte) string { + hash := md5.Sum(data) + return fmt.Sprintf("%x", hash) +} + +func compressByBZip2(originalData []byte) ([]byte, error) { + var buf bytes.Buffer + w, err := bzip2.NewWriter(&buf, &bzip2.WriterConfig{Level: bzip2.BestCompression}) + if err != nil { + return nil, err + } + if _, err := w.Write(originalData); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/endtoend/go.mod b/endtoend/go.mod new file mode 100644 index 0000000000..808649da8f --- /dev/null +++ b/endtoend/go.mod @@ -0,0 +1,16 @@ +module github.com/wesql/wescale/endtoend + +go 1.23.3 + +require ( + github.com/dsnet/compress v0.0.1 + github.com/go-sql-driver/mysql v1.8.1 + github.com/stretchr/testify v1.10.0 +) + +require ( + filippo.io/edwards25519 v1.1.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/endtoend/go.sum b/endtoend/go.sum new file mode 100644 index 0000000000..3cfda2e05c --- /dev/null +++ b/endtoend/go.sum @@ -0,0 +1,20 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/endtoend/wasm/cleanup.sql b/endtoend/wasm/cleanup.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/endtoend/wasm/main_test.go b/endtoend/wasm/main_test.go new file mode 100644 index 0000000000..d1b5fdba3d --- /dev/null +++ b/endtoend/wasm/main_test.go @@ -0,0 +1,44 @@ +package wasm + +import ( + _ "embed" + "flag" + "github.com/wesql/wescale/endtoend/framework/clusters" + "log" + "os" + "testing" +) + +var dbName = "wasm_e2e_test" + +//go:embed setup.sql +var setupSql string + +//go:embed cleanup.sql +var cleanupSql string + +var cluster *clusters.SingleNodeCluster + +func TestMain(m *testing.M) { + cluster = clusters.NewDefaultSingleNodeCluster() + // Register flags for the single node cluster, allowing the user to override the default values + cluster.RegisterFlagsForSingleNodeCluster() + flag.Parse() + + // Setup the test environment + err := cluster.SetUp(dbName, setupSql, cleanupSql) + if err != nil { + log.Fatalf("Setup failed: %v", err) + } + + // Run the tests + code := m.Run() + + // Cleanup the test environment + err = cluster.CleanUp() + if err != nil { + log.Fatalf("Cleanup failed: %v", err) + } + + os.Exit(code) +} diff --git a/endtoend/wasm/setup.sql b/endtoend/wasm/setup.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/endtoend/wasm/testdata/datamasking.wasm b/endtoend/wasm/testdata/datamasking.wasm new file mode 100644 index 0000000000..ceb2ccaa3b Binary files /dev/null and b/endtoend/wasm/testdata/datamasking.wasm differ diff --git a/endtoend/wasm/testdata/interceptor.wasm b/endtoend/wasm/testdata/interceptor.wasm new file mode 100644 index 0000000000..b7302c04ba Binary files /dev/null and b/endtoend/wasm/testdata/interceptor.wasm differ diff --git a/endtoend/wasm/wasm_test.go b/endtoend/wasm/wasm_test.go new file mode 100644 index 0000000000..a59e54e903 --- /dev/null +++ b/endtoend/wasm/wasm_test.go @@ -0,0 +1,68 @@ +package wasm + +import ( + _ "embed" + "github.com/stretchr/testify/assert" + "github.com/wesql/wescale/endtoend/framework" + "testing" +) + +//go:embed testdata/datamasking.wasm +var dataMaskingBytes []byte + +func TestDataMaskingPlugin(t *testing.T) { + filterName := "datamasking" + wasmName := "datamasking" + framework.InstallWasm(t, dataMaskingBytes, wasmName, cluster.WescaleDb) + defer framework.UninstallWasm(t, wasmName, cluster.WescaleDb) + + filterBuilder := framework.NewWasmFilterBuilder(filterName, wasmName).SetPlans("Select") + framework.CreateFilterIfNotExists(t, filterBuilder, cluster.WescaleDb) + defer framework.DropFilter(t, filterName, cluster.WescaleDb) + framework.AssertFilterExists(t, filterName, cluster.WescaleDb) + + framework.ExecNoError(t, cluster.WescaleDb, "create table wasm_e2e_test.test (id int, name varchar(255))") + defer framework.ExecNoError(t, cluster.WescaleDb, "drop table wasm_e2e_test.test") + framework.ExecNoError(t, cluster.WescaleDb, "insert into wasm_e2e_test.test values (1, 'test')") + rows := framework.QueryNoError(t, cluster.WescaleDb, "select * from wasm_e2e_test.test") + defer rows.Close() + for rows.Next() { + var id int + var name string + err := rows.Scan(&id, &name) + assert.NoError(t, err) + assert.NotEqual(t, 1, id) + assert.Equal(t, "****", name) + } +} + +//go:embed testdata/interceptor.wasm +var interceptorBytes []byte + +func TestInterceptorPlugin(t *testing.T) { + filterName := "interceptor" + wasmName := "interceptor" + framework.InstallWasm(t, interceptorBytes, wasmName, cluster.WescaleDb) + defer framework.UninstallWasm(t, wasmName, cluster.WescaleDb) + + filterBuilder := framework.NewWasmFilterBuilder(filterName, wasmName).SetPlans("Update,Delete") + framework.CreateFilterIfNotExists(t, filterBuilder, cluster.WescaleDb) + defer framework.DropFilter(t, filterName, cluster.WescaleDb) + framework.AssertFilterExists(t, filterName, cluster.WescaleDb) + + framework.ExecNoError(t, cluster.WescaleDb, "create table wasm_e2e_test.test (id int, name varchar(255))") + defer framework.ExecNoError(t, cluster.WescaleDb, "drop table wasm_e2e_test.test") + framework.ExecNoError(t, cluster.WescaleDb, "insert into wasm_e2e_test.test values (1, 'test')") + rows := framework.QueryNoError(t, cluster.WescaleDb, "select * from wasm_e2e_test.test") + defer rows.Close() + + // Expect the row to be unchanged + framework.ExecWithErrorContains(t, cluster.WescaleDb, "no where clause", "update wasm_e2e_test.test set name = 'test2'") + // Expect the row to be unchanged + framework.ExecNoError(t, cluster.WescaleDb, "update wasm_e2e_test.test set name = 'test2' where id = 1") + + // Expect the row to be unchanged + framework.ExecWithErrorContains(t, cluster.WescaleDb, "no where clause", "delete from wasm_e2e_test.test") + // Expect the row to be deleted + framework.ExecNoError(t, cluster.WescaleDb, "delete from wasm_e2e_test.test where id = 1") +} diff --git a/examples/wesql-server/init_single_node_cluster.sh b/examples/wesql-server/init_single_node_cluster.sh index 9c1418d588..7f1e58a3ff 100755 --- a/examples/wesql-server/init_single_node_cluster.sh +++ b/examples/wesql-server/init_single_node_cluster.sh @@ -1,6 +1,9 @@ #!/bin/bash set -m +# Get the directory where the script is located +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + if [ "$(id -u)" -eq 0 ]; then exec su -s /bin/bash vitess "$0" "$@" fi @@ -8,13 +11,15 @@ fi export START_VTTABLET=${START_VTTABLET:-1} export START_VTGATE=${START_VTGATE:-1} +export CONFIG_PATH=${CONFIG_PATH:-"$SCRIPT_DIR/../../config/wescale/default"} + # Set default values export MYSQL_ROOT_USER=${MYSQL_ROOT_USER:-'root'} export MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-'passwd'} export MYSQL_HOST=${MYSQL_HOST:-'127.0.0.1'} export MYSQL_PORT=${MYSQL_PORT:-'3306'} -export VTDATAROOT=${VTDATAROOT:-$(pwd)/vtdataroot} +export VTDATAROOT=${VTDATAROOT:-$SCRIPT_DIR/vtdataroot} # Define a function to catch script exit signals and clean up background processes cleanup() { @@ -36,50 +41,27 @@ run_with_prefix() { "$@" 2>&1 | sed "s/^/[$prefix] /" & } -# Function to wait for MySQL to become available using mysql command -wait_for_mysql() { - echo "Checking MySQL connection..." - local MAX_WAIT_TIME=600 # Maximum wait time in seconds (5 minutes) - local WAIT_INTERVAL=5 # Interval between connection attempts in seconds - local ELAPSED_TIME=0 - - while true; do - if mysql -h"$MYSQL_HOST" -P"$MYSQL_PORT" -u"$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "SELECT 1;" >/dev/null 2>&1; then - echo "MySQL is available." - break - else - echo "MySQL is not available yet. Waiting..." - sleep $WAIT_INTERVAL - ELAPSED_TIME=$((ELAPSED_TIME + WAIT_INTERVAL)) - if [ $ELAPSED_TIME -ge $MAX_WAIT_TIME ]; then - echo "Error: Unable to connect to MySQL after $MAX_WAIT_TIME seconds." - exit 1 - fi - fi - done -} - # Wait for MySQL to be available -./wait-for-service.sh mysql $MYSQL_HOST $MYSQL_PORT +"$SCRIPT_DIR/wait-for-service.sh" mysql $MYSQL_HOST $MYSQL_PORT echo "Initializing single-node cluster..." # Start etcd echo "Starting etcd..." -run_with_prefix "etcd" ./etcd.sh +run_with_prefix "etcd" "$SCRIPT_DIR/etcd.sh" echo "Waiting for etcd service to start..." -./wait-for-service.sh etcd 127.0.0.1 2379 +"$SCRIPT_DIR/wait-for-service.sh" etcd 127.0.0.1 2379 echo "etcd has started." # etcd post-start configuration echo "Executing etcd post-start configuration..." -./etcd-post-start.sh +"$SCRIPT_DIR/etcd-post-start.sh" # Start vtctld echo "Starting vtctld..." -run_with_prefix "vtctld" ./vtctld.sh +run_with_prefix "vtctld" "$SCRIPT_DIR/vtctld.sh" echo "Waiting for vtctld service to start..." -./wait-for-service.sh vtctld 127.0.0.1 15999 +"$SCRIPT_DIR/wait-for-service.sh" vtctld 127.0.0.1 15999 echo "vtctld has started." # Start vttablet 0 @@ -88,9 +70,9 @@ if [ "$START_VTTABLET" -eq 1 ]; then export VTTABLET_PORT=${VTTABLET_PORT:-'15100'} export VTTABLET_GRPC_PORT=${VTTABLET_GRPC_PORT:-'16100'} echo "Starting vttablet..." - run_with_prefix "vttablet" ./vttablet.sh + run_with_prefix "vttablet" "$SCRIPT_DIR/vttablet.sh" echo "Waiting for vttablet service to start..." - ./wait-for-service.sh vttablet 127.0.0.1 $VTTABLET_GRPC_PORT + "$SCRIPT_DIR/wait-for-service.sh" vttablet 127.0.0.1 $VTTABLET_GRPC_PORT echo "vttablet has started." fi @@ -100,9 +82,9 @@ if [ "$START_VTGATE" -eq 1 ]; then export VTGATE_GRPC_PORT=${VTGATE_GRPC_PORT:-'15991'} export VTGATE_MYSQL_PORT=${VTGATE_MYSQL_PORT:-'15306'} echo "Starting vtgate..." - run_with_prefix "vtgate" ./vtgate.sh + run_with_prefix "vtgate" "$SCRIPT_DIR/vtgate.sh" echo "Waiting for vtgate service to start..." - ./wait-for-service.sh vtgate 127.0.0.1 $VTGATE_MYSQL_PORT + "$SCRIPT_DIR/wait-for-service.sh" vtgate 127.0.0.1 $VTGATE_MYSQL_PORT echo "vtgate has started." echo " @@ -114,7 +96,5 @@ if [ "$START_VTGATE" -eq 1 ]; then " fi - - # Keep the script running to catch exit signals wait \ No newline at end of file diff --git a/examples/wesql-server/vtgate.sh b/examples/wesql-server/vtgate.sh index a7920713d1..d01e449386 100755 --- a/examples/wesql-server/vtgate.sh +++ b/examples/wesql-server/vtgate.sh @@ -7,6 +7,7 @@ web_port=${VTGATE_WEB_PORT:-'15001'} grpc_port=${VTGATE_GRPC_PORT:-'15991'} mysql_server_port=${VTGATE_MYSQL_PORT:-'15306'} mysql_server_socket_path="/tmp/mysql.sock" +config_path=${CONFIG_PATH} topology_flags=${TOPOLOGY_FLAGS:-'--topo_implementation etcd2 --topo_global_server_address 127.0.0.1:2379 --topo_global_root /vitess/global'} @@ -27,4 +28,4 @@ vtgate \ --cells_to_watch $cell \ --service_map 'grpc-vtgateservice' \ --pid_file $VTDATAROOT/vtgate.pid \ - --config_path ./conf + --config_path $config_path diff --git a/examples/wesql-server/vttablet.sh b/examples/wesql-server/vttablet.sh index 5432e00e69..2ca3385a81 100755 --- a/examples/wesql-server/vttablet.sh +++ b/examples/wesql-server/vttablet.sh @@ -13,6 +13,7 @@ grpc_port=${VTTABLET_GRPC_PORT:-'16100'} vtctld_host=${VTCTLD_HOST:-'127.0.0.1'} vtctld_web_port=${VTCTLD_WEB_PORT:-'15000'} tablet_hostname='127.0.0.1' +config_path=${CONFIG_PATH} printf -v alias '%s-%010d' $cell $uid printf -v tablet_dir 'vt_%010d' $uid @@ -51,4 +52,4 @@ vttablet \ --pid_file $VTDATAROOT/vttablet.pid \ --vtctld_addr http://$vtctld_host:$vtctld_web_port/ \ --disable_active_reparents \ - --config_path ./conf + --config_path $config_path