From cea10d194ea87d76cdd52bd63708ae54342145d8 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Fri, 27 Aug 2021 10:10:32 +0300 Subject: [PATCH 01/20] feat(vitess-cdc): Add vitess cdc connector --- flink-connector-vitess-cdc/pom.xml | 160 ++++++++++++ .../cdc/connectors/vitess/VitessSource.java | 239 ++++++++++++++++++ .../connectors/vitess/VitessValidator.java | 45 ++++ .../connectors/vitess/config/TabletType.java | 41 +++ .../vitess/config/VtctldConfig.java | 75 ++++++ .../vitess/table/VitessTableFactory.java | 197 +++++++++++++++ .../vitess/table/VitessTableSource.java | 186 ++++++++++++++ .../org.apache.flink.table.factories.Factory | 16 ++ .../{postgres => vitess}/DummyDocs.java | 2 +- flink-sql-connector-vitess-cdc/pom.xml | 78 ++++++ .../cdc/connectors/vitess/DummyDocs.java | 22 ++ pom.xml | 10 +- 12 files changed, 1062 insertions(+), 9 deletions(-) create mode 100644 flink-connector-vitess-cdc/pom.xml create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/TabletType.java create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java create mode 100644 flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java create mode 100644 flink-connector-vitess-cdc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory rename flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/{postgres => vitess}/DummyDocs.java (93%) create mode 100644 flink-sql-connector-vitess-cdc/pom.xml create mode 100644 flink-sql-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml new file mode 100644 index 00000000000..7defc415c4c --- /dev/null +++ b/flink-connector-vitess-cdc/pom.xml @@ -0,0 +1,160 @@ + + + + + flink-cdc-connectors + com.ververica + 2.1-SNAPSHOT + + 4.0.0 + + flink-connector-vitess-cdc + flink-connector-vitess-cdc + jar + + + + + + com.ververica + flink-connector-debezium + ${project.version} + + + kafka-log4j-appender + org.apache.kafka + + + + + + io.debezium + debezium-connector-vitess + ${debezium.version} + + + + + + com.ververica + flink-connector-test-util + ${project.version} + test + + + + io.debezium + debezium-core + ${debezium.version} + test-jar + test + + + + + + org.apache.flink + flink-table-planner-blink_${scala.binary.version} + ${flink.version} + test + + + + org.apache.flink + flink-table-runtime-blink_${scala.binary.version} + ${flink.version} + test + + + + org.apache.flink + flink-test-utils_${scala.binary.version} + ${flink.version} + test + + + + org.apache.flink + flink-core + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-streaming-java_${scala.binary.version} + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-table-common + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-tests + ${flink.version} + test-jar + test + + + + org.apache.flink + flink-table-planner-blink_${scala.binary.version} + ${flink.version} + test-jar + test + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + test + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + 1 + -Xms256m -Xmx2048m -Dlog4j.configurationFile=${log4j.configuration} + -Dmvn.forkNumber=${surefire.forkNumber} -XX:-UseGCOverheadLimit + + + + + + + \ No newline at end of file diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java new file mode 100644 index 00000000000..05e920cd079 --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java @@ -0,0 +1,239 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess; + +import com.ververica.cdc.connectors.vitess.config.TabletType; +import com.ververica.cdc.connectors.vitess.config.VtctldConfig; +import com.ververica.cdc.debezium.DebeziumDeserializationSchema; +import com.ververica.cdc.debezium.DebeziumSourceFunction; +import io.debezium.connector.vitess.VitessConnector; + +import java.util.Properties; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A builder to build a SourceFunction which can read and process vitess database changes. The + * Vitess connector subscribes to VTGate's VStream gRPC service. VTGate is a lightweight, stateless + * gRPC server, which is part of the Vitess cluster setup. + */ +public class VitessSource { + + public static Builder builder() { + return new Builder<>(); + } + + /** Builder class of {@link VitessSource}. */ + public static class Builder { + + private String pluginName = "decoderbufs"; + private String slotName = "flink"; + private int port = 15991; // default 15991 port + private String hostname; + private String keyspace; + private String username; + private String password; + private VtctldConfig vtctldConfig; + private TabletType tabletType = TabletType.REPLICA; + private String[] tableIncludeList; + private String[] tableExcludeList; + private String[] columnIncludeList; + private String[] columnExcludeList; + private Properties dbzProperties; + private DebeziumDeserializationSchema deserializer; + + /** + * The name of the Vitess logical decoding plug-in installed on the server. Supported values + * are decoderbufs + */ + public Builder decodingPluginName(String name) { + this.pluginName = name; + return this; + } + + /** Hostname of the VTGate’s VStream server. */ + public Builder hostname(String hostname) { + this.hostname = hostname; + return this; + } + + /** Integer port number of the VTGate’s VStream server. */ + public Builder port(int port) { + this.port = port; + return this; + } + + /** + * The name of the keyspace (a.k.a database). If no shard is specified, it reads change + * events from all shards in the keyspace. + */ + public Builder keyspace(String keyspace) { + this.keyspace = keyspace; + return this; + } + + /** VTCtld server config. */ + public Builder vtctldConfig(VtctldConfig vtctldConfig) { + this.vtctldConfig = vtctldConfig; + return this; + } + + /** + * The type of Tablet (hence MySQL) from which to stream the changes: MASTER represents + * streaming from the master MySQL instance REPLICA represents streaming from the replica + * slave MySQL instance RDONLY represents streaming from the read-only slave MySQL instance. + */ + public Builder tabletType(TabletType tabletType) { + this.tabletType = tabletType; + return this; + } + + /** The username of the Vitess database server (VTGate gRPC). */ + public Builder username(String username) { + this.username = username; + return this; + } + + /** The password of the Vitess database server (VTGate gRPC). */ + public Builder password(String password) { + this.password = password; + return this; + } + + /** + * The name of the Vitess logical decoding slot that was created for streaming changes from + * a particular plug-in for a particular database/schema. The server uses this slot to + * stream events to the connector that you are configuring. Default is "flink". + */ + public Builder slotName(String slotName) { + this.slotName = slotName; + return this; + } + + /** + * An optional, comma-separated list of regular expressions that match fully-qualified table + * identifiers for tables whose changes you want to capture. Any table not included in + * table.include.list does not have its changes captured. Each identifier is of the form + * keyspace.tableName. By default, the connector captures changes in every non-system table + * in each schema whose changes are being captured. Do not also set the table.exclude.list + * property. + */ + public Builder tableIncludeList(String... tableIncludeList) { + this.tableIncludeList = tableIncludeList; + return this; + } + + /** + * An optional, comma-separated list of regular expressions that match fully-qualified table + * identifiers for tables whose changes you do not want to capture. Any table not included + * in table.exclude.list has it changes captured. Each identifier is of the form + * keyspace.tableName. Do not also set the table.include.list property. + */ + public Builder tableExcludeList(String... tableExcludeList) { + this.tableExcludeList = tableExcludeList; + return this; + } + + /** + * An optional, comma-separated list of regular expressions that match the fully-qualified + * names of columns that should be included in change event record values. Fully-qualified + * names for columns are of the form keyspace.tableName.columnName. Do not also set the + * column.exclude.list property. + */ + public Builder columnIncludeList(String... columnIncludeList) { + this.columnIncludeList = columnIncludeList; + return this; + } + + /** + * An optional, comma-separated list of regular expressions that match the fully-qualified + * names of columns that should be excluded from change event record values. Fully-qualified + * names for columns are of the form keyspace.tableName.columnName. Do not also set the + * column.include.list property. + */ + public Builder columnExcludeList(String... columnExcludeList) { + this.columnExcludeList = columnExcludeList; + return this; + } + + /** The Debezium Vitess connector properties. */ + public Builder debeziumProperties(Properties properties) { + this.dbzProperties = properties; + return this; + } + + /** + * The deserializer used to convert from consumed {@link + * org.apache.kafka.connect.source.SourceRecord}. + */ + public Builder deserializer(DebeziumDeserializationSchema deserializer) { + this.deserializer = deserializer; + return this; + } + + public DebeziumSourceFunction build() { + Properties props = new Properties(); + props.setProperty("connector.class", VitessConnector.class.getCanonicalName()); + props.setProperty("plugin.name", pluginName); + props.setProperty("slot.name", slotName); + // hard code server name, because we don't need to distinguish it, docs: + // Logical name that identifies and provides a namespace for the particular Vitess + // Vtgate server/cluster being monitored. The logical name should be unique across + // all other connectors, since it is used as a prefix for all Kafka topic names coming + // from this connector. Only alphanumeric characters and underscores should be used. + props.setProperty("database.server.name", "vitess_cdc_source"); + props.setProperty("database.hostname", checkNotNull(hostname)); + props.setProperty("database.port", String.valueOf(port)); + props.setProperty("database.user", checkNotNull(username)); + props.setProperty("database.password", checkNotNull(password)); + props.setProperty("vitess.keyspace", checkNotNull(keyspace)); + + props.setProperty("vitess.vtctld.host", checkNotNull(vtctldConfig.hostname)); + props.setProperty("vitess.vtctld.port", String.valueOf(vtctldConfig.port)); + props.setProperty("vitess.vtctld.user", vtctldConfig.username); + props.setProperty("vitess.vtctld.password", vtctldConfig.password); + props.setProperty("vitess.tablet.type", tabletType.name()); + props.setProperty("vitess.shard", tabletType.name()); + + // The maximum number of tasks that should be created for this connector. + // The Vitess connector always uses a single task and therefore does not use this value, + // so the default is always acceptable. + props.setProperty("tasks.max", "1"); + + if (tableIncludeList != null) { + props.setProperty("table.include.list", String.join(",", tableIncludeList)); + } + if (tableExcludeList != null) { + props.setProperty("table.exclude.list", String.join(",", tableExcludeList)); + } + if (columnIncludeList != null) { + props.setProperty("column.include.list", String.join(",", columnIncludeList)); + } + if (columnExcludeList != null) { + props.setProperty("column.exclude.list", String.join(",", columnExcludeList)); + } + if (dbzProperties != null) { + dbzProperties.forEach(props::put); + } + + return new DebeziumSourceFunction<>( + deserializer, props, null, new VitessValidator(props)); + } + } +} diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java new file mode 100644 index 00000000000..0542616f1e3 --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess; + +import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; + +import com.ververica.cdc.debezium.Validator; +import io.debezium.connector.vitess.VitessConnector; + +import java.util.Map; +import java.util.Properties; + +/** The validator for Vitess. */ +public class VitessValidator implements Validator { + + private static final long serialVersionUID = 1L; + + private final Map configuration; + + public VitessValidator(Properties properties) { + this.configuration = Maps.fromProperties(properties); + } + + @Override + public void validate() { + VitessConnector c = new VitessConnector(); + c.validate(configuration); + } +} diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/TabletType.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/TabletType.java new file mode 100644 index 00000000000..a4bd79ebd51 --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/TabletType.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess.config; + +/** The type of Tablet (hence MySQL) from which to stream the changes. */ +public enum TabletType { + /** Streaming from the master MySQL instance. */ + MASTER, + /** Streaming from the replica slave MySQL instance. */ + REPLICA, + /** Streaming from the read-only slave MySQL instance. */ + RDONLY; + + public static TabletType master() { + return TabletType.MASTER; + } + + public static TabletType replica() { + return TabletType.REPLICA; + } + + public static TabletType rdonly() { + return TabletType.RDONLY; + } +} diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java new file mode 100644 index 00000000000..54bfc72f723 --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess.config; + +/** VTCtld server configuration options. */ +public class VtctldConfig { + + public String hostname; + public int port = 15999; // default 15999 port + public String username; + public String password; + + /** Builder class of {@link VtctldConfig}. */ + public static final class Builder { + private String hostname; + private int port = 15999; // default 15999 port + private String username; + private String password; + + /** IP address or hostname of the VTCtld server. */ + public Builder hostname(String hostname) { + this.hostname = hostname; + return this; + } + + /** Integer port number of the VTCtld server. */ + public Builder port(int port) { + this.port = port; + return this; + } + + /** + * An optional username of the VTCtld server. If not configured, unauthenticated VTCtld gRPC + * is used. + */ + public Builder username(String username) { + this.username = username; + return this; + } + + /** + * An optional password of the VTCtld server. If not configured, unauthenticated VTCtld gRPC + * is used. + */ + public Builder password(String password) { + this.password = password; + return this; + } + + public VtctldConfig build() { + VtctldConfig vtctldConfig = new VtctldConfig(); + vtctldConfig.password = this.password; + vtctldConfig.username = this.username; + vtctldConfig.hostname = this.hostname; + vtctldConfig.port = this.port; + return vtctldConfig; + } + } +} diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java new file mode 100644 index 00000000000..ae5af87e18a --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess.table; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.DynamicTableSourceFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.utils.TableSchemaUtils; + +import com.ververica.cdc.connectors.vitess.config.TabletType; +import com.ververica.cdc.connectors.vitess.config.VtctldConfig; + +import java.util.HashSet; +import java.util.Set; + +import static com.ververica.cdc.debezium.table.DebeziumOptions.DEBEZIUM_OPTIONS_PREFIX; +import static com.ververica.cdc.debezium.table.DebeziumOptions.getDebeziumProperties; + +/** Factory for creating configured instance of {@link VitessTableSource}. */ +public class VitessTableFactory implements DynamicTableSourceFactory { + + private static final String IDENTIFIER = "vitess-cdc"; + + private static final ConfigOption HOSTNAME = + ConfigOptions.key("hostname") + .stringType() + .noDefaultValue() + .withDescription("Hostname of the VTGate’s VStream server."); + + private static final ConfigOption PORT = + ConfigOptions.key("port") + .intType() + .defaultValue(15991) + .withDescription("Integer port number of the VTGate’s VStream server."); + + private static final ConfigOption KEYSPACE = + ConfigOptions.key("keyspace") + .stringType() + .noDefaultValue() + .withDescription( + "The name of the keyspace (a.k.a database). If no shard is specified, it reads change events from all shards in the keyspace."); + + private static final ConfigOption USERNAME = + ConfigOptions.key("username") + .stringType() + .noDefaultValue() + .withDescription("The username of the Vitess database server (VTGate gRPC)."); + + private static final ConfigOption PASSWORD = + ConfigOptions.key("password") + .stringType() + .noDefaultValue() + .withDescription("The password of the Vitess database server (VTGate gRPC)."); + + private static final ConfigOption VTCTL_HOSTNAME = + ConfigOptions.key("vtctl.hostname") + .stringType() + .noDefaultValue() + .withDescription("IP address or hostname of the VTCtld server."); + + private static final ConfigOption VTCTL_PORT = + ConfigOptions.key("vtctl.port") + .intType() + .defaultValue(15999) + .withDescription("Integer port number of the VTCtld server."); + + private static final ConfigOption VTCTL_USERNAME = + ConfigOptions.key("vtctl.username") + .stringType() + .noDefaultValue() + .withDescription("The username of the Vitess VTCtld server."); + + private static final ConfigOption VTCTL_PASSWORD = + ConfigOptions.key("vtctl.password") + .stringType() + .noDefaultValue() + .withDescription("The password of the Vitess VTCtld server."); + + private static final ConfigOption TABLET_TYPE = + ConfigOptions.key("tablet-type") + .stringType() + .defaultValue(TabletType.REPLICA.name()) + .withDescription( + "The type of Tablet (hence MySQL) from which to stream the changes:"); + + private static final ConfigOption TABLE_NAME = + ConfigOptions.key("table-name") + .stringType() + .noDefaultValue() + .withDescription("Table name of the MYSQL database to monitor."); + + private static final ConfigOption DECODING_PLUGIN_NAME = + ConfigOptions.key("decoding.plugin.name") + .stringType() + .defaultValue("decoderbufs") + .withDescription( + "The name of the Vitess logical decoding plug-in installed on the server."); + + private static final ConfigOption SLOT_NAME = + ConfigOptions.key("slot.name") + .stringType() + .defaultValue("flink") + .withDescription( + "The name of the Vitess logical decoding slot that was created for streaming changes " + + "from a particular plug-in for a particular database/schema. The server uses this slot " + + "to stream events to the connector that you are configuring. Default is \"flink\"."); + + @Override + public DynamicTableSource createDynamicTableSource(DynamicTableFactory.Context context) { + final FactoryUtil.TableFactoryHelper helper = + FactoryUtil.createTableFactoryHelper(this, context); + helper.validateExcept(DEBEZIUM_OPTIONS_PREFIX); + + final ReadableConfig config = helper.getOptions(); + String hostname = config.get(HOSTNAME); + int port = config.get(PORT); + String keyspace = config.get(KEYSPACE); + String tableName = config.get(TABLE_NAME); + String username = config.get(USERNAME); + String password = config.get(PASSWORD); + VtctldConfig vtctldConfig = + new VtctldConfig.Builder() + .hostname(config.get(VTCTL_HOSTNAME)) + .port(config.get(VTCTL_PORT)) + .username(config.get(VTCTL_USERNAME)) + .password(config.get(VTCTL_PASSWORD)) + .build(); + TabletType tabletType = TabletType.valueOf(config.get(TABLET_TYPE)); + String pluginName = config.get(DECODING_PLUGIN_NAME); + String slotName = config.get(SLOT_NAME); + TableSchema physicalSchema = + TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); + + return new VitessTableSource( + physicalSchema, + port, + hostname, + keyspace, + tableName, + username, + password, + vtctldConfig, + tabletType, + pluginName, + slotName, + getDebeziumProperties(context.getCatalogTable().getOptions())); + } + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + Set> options = new HashSet<>(); + options.add(HOSTNAME); + options.add(KEYSPACE); + options.add(USERNAME); + options.add(PASSWORD); + options.add(VTCTL_HOSTNAME); + options.add(TABLE_NAME); + return options; + } + + @Override + public Set> optionalOptions() { + Set> options = new HashSet<>(); + options.add(PORT); + options.add(VTCTL_PORT); + options.add(TABLET_TYPE); + options.add(DECODING_PLUGIN_NAME); + options.add(SLOT_NAME); + return options; + } +} diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java new file mode 100644 index 00000000000..b20f4d495b5 --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess.table; + +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.connector.source.ScanTableSource; +import org.apache.flink.table.connector.source.SourceFunctionProvider; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.types.RowKind; + +import com.ververica.cdc.connectors.vitess.VitessSource; +import com.ververica.cdc.connectors.vitess.config.TabletType; +import com.ververica.cdc.connectors.vitess.config.VtctldConfig; +import com.ververica.cdc.debezium.DebeziumDeserializationSchema; +import com.ververica.cdc.debezium.DebeziumSourceFunction; +import com.ververica.cdc.debezium.table.RowDataDebeziumDeserializeSchema; + +import java.time.ZoneId; +import java.util.Objects; +import java.util.Properties; + +import static org.apache.flink.util.Preconditions.checkNotNull; + +/** + * A {@link DynamicTableSource} that describes how to create a Vitess source from a logical + * description. + */ +public class VitessTableSource implements ScanTableSource { + + private final TableSchema physicalSchema; + private String pluginName = "decoderbufs"; + private String slotName = "flink"; + private int port = 15991; // default 15991 port + private String hostname; + private String keyspace; + private String username; + private String password; + private String tableName; + private VtctldConfig vtctldConfig; + private TabletType tabletType; + private Properties dbzProperties; + + public VitessTableSource( + TableSchema physicalSchema, + int port, + String hostname, + String keyspace, + String tableName, + String username, + String password, + VtctldConfig vtctldConfig, + TabletType tabletType, + String pluginName, + String slotName, + Properties dbzProperties) { + this.physicalSchema = physicalSchema; + this.port = port; + this.hostname = checkNotNull(hostname); + this.keyspace = checkNotNull(keyspace); + this.tableName = checkNotNull(tableName); + this.username = checkNotNull(username); + this.password = checkNotNull(password); + this.pluginName = checkNotNull(pluginName); + this.slotName = slotName; + this.dbzProperties = dbzProperties; + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.newBuilder() + .addContainedKind(RowKind.INSERT) + .addContainedKind(RowKind.UPDATE_BEFORE) + .addContainedKind(RowKind.UPDATE_AFTER) + .addContainedKind(RowKind.DELETE) + .build(); + } + + @Override + public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { + RowType rowType = (RowType) physicalSchema.toRowDataType().getLogicalType(); + TypeInformation typeInfo = + scanContext.createTypeInformation(physicalSchema.toRowDataType()); + DebeziumDeserializationSchema deserializer = + new RowDataDebeziumDeserializeSchema( + rowType, typeInfo, ((rowData, rowKind) -> {}), ZoneId.of("UTC")); + + DebeziumSourceFunction sourceFunction = + VitessSource.builder() + .hostname(hostname) + .port(port) + .keyspace(keyspace) + .tableIncludeList(tableName) + .username(username) + .password(password) + .tabletType(tabletType) + .decodingPluginName(pluginName) + .vtctldConfig(vtctldConfig) + .slotName(slotName) + .debeziumProperties(dbzProperties) + .deserializer(deserializer) + .build(); + return SourceFunctionProvider.of(sourceFunction, false); + } + + @Override + public DynamicTableSource copy() { + return new VitessTableSource( + physicalSchema, + port, + hostname, + keyspace, + tableName, + username, + password, + vtctldConfig, + tabletType, + pluginName, + slotName, + dbzProperties); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + VitessTableSource that = (VitessTableSource) o; + return port == that.port + && Objects.equals(physicalSchema, that.physicalSchema) + && Objects.equals(pluginName, that.pluginName) + && Objects.equals(slotName, that.slotName) + && Objects.equals(hostname, that.hostname) + && Objects.equals(keyspace, that.keyspace) + && Objects.equals(username, that.username) + && Objects.equals(password, that.password) + && Objects.equals(tableName, that.tableName) + && Objects.equals(vtctldConfig, that.vtctldConfig) + && tabletType == that.tabletType + && Objects.equals(dbzProperties, that.dbzProperties); + } + + @Override + public int hashCode() { + return Objects.hash( + physicalSchema, + pluginName, + slotName, + port, + hostname, + keyspace, + username, + password, + tableName, + vtctldConfig, + tabletType, + dbzProperties); + } + + @Override + public String asSummaryString() { + return "Vitess-CDC"; + } +} diff --git a/flink-connector-vitess-cdc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-connector-vitess-cdc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 00000000000..5cb7d68c2fc --- /dev/null +++ b/flink-connector-vitess-cdc/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +com.ververica.cdc.connectors.vitess.table.VitessTableFactory \ No newline at end of file diff --git a/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java b/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java similarity index 93% rename from flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java rename to flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java index deb9e330d14..46c174dc6ed 100644 --- a/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java +++ b/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.ververica.cdc.connectors.postgres; +package com.ververica.cdc.connectors.vitess; /** This is used to generate a dummy docs jar for this module to pass OSS repository rule. */ public class DummyDocs {} diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml new file mode 100644 index 00000000000..e7176141aa4 --- /dev/null +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -0,0 +1,78 @@ + + + + + flink-cdc-connectors + com.ververica + 2.1-SNAPSHOT + + 4.0.0 + + flink-sql-connector-vitess-cdc + + + + com.ververica + flink-connector-vitess-cdc + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + shade-flink + package + + shade + + + false + + + *:* + + + + + org.apache.kafka:* + + kafka/kafka-version.properties + LICENSE + + NOTICE + common/** + + + + + + + + + + \ No newline at end of file diff --git a/flink-sql-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java b/flink-sql-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java new file mode 100644 index 00000000000..546ee208eaf --- /dev/null +++ b/flink-sql-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ververica.cdc.connectors.vitess; + +/** This is used to generate a dummy docs jar for this module to pass OSS repository rule. */ +public class DummyDocs {} diff --git a/pom.xml b/pom.xml index 91d144e0ba7..2936044a872 100644 --- a/pom.xml +++ b/pom.xml @@ -39,6 +39,7 @@ under the License. flink-connector-oracle-cdc flink-connector-mongodb-cdc flink-connector-oceanbase-cdc + flink-connector-vitess-cdc flink-connector-sqlserver-cdc flink-connector-tidb-cdc flink-connector-db2-cdc @@ -47,6 +48,7 @@ under the License. flink-sql-connector-mongodb-cdc flink-sql-connector-oracle-cdc flink-sql-connector-oceanbase-cdc + flink-sql-connector-vitess-cdc flink-sql-connector-sqlserver-cdc flink-sql-connector-tidb-cdc flink-sql-connector-db2-cdc @@ -87,7 +89,6 @@ under the License. ${java.version} ${java.version} 1.3 - 4.0.1 1.7.15 2.17.1 2.4.2 @@ -179,13 +180,6 @@ under the License. ${log4j.version} test - - - org.awaitility - awaitility - jar - ${version.awaitility} - From f349b5fd6636f17e6401290544ef707b0a988e90 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Fri, 27 Aug 2021 12:03:37 +0300 Subject: [PATCH 02/20] fix(vitess-cdc): Make validator serializable --- .../cdc/connectors/vitess/VitessSource.java | 15 ++++++---- .../connectors/vitess/VitessValidator.java | 3 +- .../vitess/config/VtctldConfig.java | 28 ++++++++++++++++--- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java index 05e920cd079..15b0bc04eb8 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java @@ -203,13 +203,16 @@ public DebeziumSourceFunction build() { props.setProperty("database.user", checkNotNull(username)); props.setProperty("database.password", checkNotNull(password)); props.setProperty("vitess.keyspace", checkNotNull(keyspace)); - - props.setProperty("vitess.vtctld.host", checkNotNull(vtctldConfig.hostname)); - props.setProperty("vitess.vtctld.port", String.valueOf(vtctldConfig.port)); - props.setProperty("vitess.vtctld.user", vtctldConfig.username); - props.setProperty("vitess.vtctld.password", vtctldConfig.password); props.setProperty("vitess.tablet.type", tabletType.name()); - props.setProperty("vitess.shard", tabletType.name()); + props.setProperty("vitess.vtctld.host", checkNotNull(vtctldConfig.getHostname())); + props.setProperty("vitess.vtctld.port", String.valueOf(vtctldConfig.getPort())); + + if (vtctldConfig.getUsername() != null) { + props.setProperty("vitess.vtctld.user", vtctldConfig.getUsername()); + } + if (vtctldConfig.getPassword() != null) { + props.setProperty("vitess.vtctld.password", vtctldConfig.getPassword()); + } // The maximum number of tasks that should be created for this connector. // The Vitess connector always uses a single task and therefore does not use this value, diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java index 0542616f1e3..95cd61356dd 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java @@ -23,11 +23,12 @@ import com.ververica.cdc.debezium.Validator; import io.debezium.connector.vitess.VitessConnector; +import java.io.Serializable; import java.util.Map; import java.util.Properties; /** The validator for Vitess. */ -public class VitessValidator implements Validator { +public class VitessValidator implements Validator, Serializable { private static final long serialVersionUID = 1L; diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java index 54bfc72f723..a9808d6415b 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java @@ -21,10 +21,30 @@ /** VTCtld server configuration options. */ public class VtctldConfig { - public String hostname; - public int port = 15999; // default 15999 port - public String username; - public String password; + private String hostname; + private int port = 15999; // default 15999 port + private String username; + private String password; + + public String getHostname() { + return hostname; + } + + public int getPort() { + return port; + } + + public String getUsername() { + return username; + } + + public String getPassword() { + return password; + } + + public static VtctldConfig.Builder builder() { + return new VtctldConfig.Builder(); + } /** Builder class of {@link VtctldConfig}. */ public static final class Builder { From 029922dcaae68a989821cf4a5e39d9f674e8d977 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Wed, 16 Mar 2022 15:34:39 +0200 Subject: [PATCH 03/20] feat[vitess-cdc]: Add tests and documentation --- docs/content/about.md | 141 ++++---- docs/content/connectors/index.md | 1 + docs/content/connectors/vitess-cdc.md | 309 +++++++++++++++++ flink-connector-vitess-cdc/pom.xml | 36 +- .../cdc/connectors/vitess/VitessSource.java | 11 +- .../vitess/config/VtctldConfig.java | 39 +++ .../vitess/table/VitessTableFactory.java | 6 +- .../vitess/table/VitessTableSource.java | 49 ++- .../connectors/vitess/VitessSourceTest.java | 310 ++++++++++++++++++ .../cdc/connectors/vitess/VitessTestBase.java | 106 ++++++ .../vitess/container/VitessContainer.java | 103 ++++++ .../vitess/table/VitessConnectorITCase.java | 275 ++++++++++++++++ .../vitess/table/VitessTableFactoryTest.java | 202 ++++++++++++ .../test/resources/ddl/column_type_test.sql | 44 +++ .../src/test/resources/ddl/inventory.sql | 24 ++ .../src/test/resources/log4j2-test.properties | 28 ++ 16 files changed, 1613 insertions(+), 71 deletions(-) create mode 100644 docs/content/connectors/vitess-cdc.md create mode 100644 flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java create mode 100644 flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java create mode 100644 flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java create mode 100644 flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java create mode 100644 flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java create mode 100644 flink-connector-vitess-cdc/src/test/resources/ddl/column_type_test.sql create mode 100644 flink-connector-vitess-cdc/src/test/resources/ddl/inventory.sql create mode 100644 flink-connector-vitess-cdc/src/test/resources/log4j2-test.properties diff --git a/docs/content/about.md b/docs/content/about.md index fb4fe11862a..00400a2b1e6 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -1,24 +1,26 @@ # Overview -CDC Connectors for Apache Flink® is a set of source connectors for Apache Flink®, ingesting changes from different databases using change data capture (CDC). -The CDC Connectors for Apache Flink® integrate Debezium as the engine to capture data changes. So it can fully leverage the ability of Debezium. See more about what is [Debezium](https://github.com/debezium/debezium). +CDC Connectors for Apache Flink® is a set of source connectors for Apache +Flink®, ingesting changes from different databases using change data capture (CDC). +The CDC Connectors for Apache Flink® integrate Debezium as the engine to capture data changes. So it can +fully leverage the ability of Debezium. See more about what is [Debezium](https://github.com/debezium/debezium). ![Flink_CDC](/_static/fig/flinkcdc.png "Flink CDC") ## Supported Connectors -| Connector | Database | Driver | -|----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------| -| [mongodb-cdc](connectors/mongodb-cdc.md) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.1 | -| [mysql-cdc](connectors/mysql-cdc.md) |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.27 | -| [oceanbase-cdc](connectors/oceanbase-cdc.md) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase) (MySQL mode): 2.x, 3.x | JDBC Driver: 5.1.4x | -| [oracle-cdc](connectors/oracle-cdc.md) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19 | Oracle Driver: 19.3.0.0 | -| [postgres-cdc](connectors/postgres-cdc.md) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12 | JDBC Driver: 42.2.12 | -| [sqlserver-cdc](connectors/sqlserver-cdc.md) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 7.2.2.jre8 | -| [tidb-cdc](connectors/tidb-cdc.md) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | -| [db2-cdc](connectors/db2-cdc.md) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | DB2 Driver: 11.5.0.0 | - -## Supported Flink Versions +| Connector | Database | Driver | +|----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| [mongodb-cdc](connectors/mongodb-cdc.md) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.1 | +| [mysql-cdc](connectors/mysql-cdc.md) |
  • [MySQL](https://dev.mysql.com/doc): 5.6, 5.7, 8.0.x
  • [RDS MySQL](https://www.aliyun.com/product/rds/mysql): 5.6, 5.7, 8.0.x
  • [PolarDB MySQL](https://www.aliyun.com/product/polardb): 5.6, 5.7, 8.0.x
  • [Aurora MySQL](https://aws.amazon.com/cn/rds/aurora): 5.6, 5.7, 8.0.x
  • [MariaDB](https://mariadb.org): 10.x
  • [PolarDB X](https://github.com/ApsaraDB/galaxysql): 2.0.1 | JDBC Driver: 8.0.27 | +| [oceanbase-cdc](connectors/oceanbase-cdc.md) |
  • [OceanBase CE](https://open.oceanbase.com): 3.1.x
  • [OceanBase EE](https://www.oceanbase.com/product/oceanbase) (MySQL mode): 2.x, 3.x | JDBC Driver: 5.1.4x | +| [oracle-cdc](connectors/oracle-cdc.md) |
  • [Oracle](https://www.oracle.com/index.html): 11, 12, 19 | Oracle Driver: 19.3.0.0 | +| [postgres-cdc](connectors/postgres-cdc.md) |
  • [PostgreSQL](https://www.postgresql.org): 9.6, 10, 11, 12 | JDBC Driver: 42.2.12 | +| [sqlserver-cdc](connectors/sqlserver-cdc.md) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 7.2.2.jre8 | +| [tidb-cdc](connectors/tidb-cdc.md) |
  • [TiDB](https://www.pingcap.com/): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | +| [db2-cdc](connectors/db2-cdc.md) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | DB2 Driver: 11.5.0.0 | +| [Vitess-cdc](connectors/vitess-cdc.md) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.16 | + The following table shows the version mapping between Flink® CDC Connectors and Flink®: | Flink® CDC Version | Flink® Version | @@ -35,8 +37,10 @@ The following table shows the version mapping between Flink® CDC Con ## Features -1. Supports reading database snapshot and continues to read binlogs with **exactly-once processing** even failures happen. -2. CDC connectors for DataStream API, users can consume changes on multiple databases and tables in a single job without Debezium and Kafka deployed. +1. Supports reading database snapshot and continues to read binlogs with **exactly-once processing** even failures + happen. +2. CDC connectors for DataStream API, users can consume changes on multiple databases and tables in a single job without + Debezium and Kafka deployed. 3. CDC connectors for Table/SQL API, users can use SQL DDL to create a CDC source to monitor changes on a single table. ## Usage for Table/SQL API @@ -48,7 +52,9 @@ We need several steps to setup a Flink cluster with the provided connector. 3. Put the downloaded jars under `FLINK_HOME/lib/`. 4. Restart the Flink cluster. -The example shows how to create a MySQL CDC source in [Flink SQL Client](https://ci.apache.org/projects/flink/flink-docs-release-1.13/dev/table/sqlClient.html) and execute queries on it. +The example shows how to create a MySQL CDC source +in [Flink SQL Client](https://ci.apache.org/projects/flink/flink-docs-release-1.13/dev/table/sqlClient.html) and execute +queries on it. ```sql -- creates a mysql cdc table source @@ -92,33 +98,35 @@ import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema; import com.ververica.cdc.connectors.mysql.source.MySqlSource; public class MySqlBinlogSourceExample { - public static void main(String[] args) throws Exception { - MySqlSource mySqlSource = MySqlSource.builder() - .hostname("yourHostname") - .port(yourPort) - .databaseList("yourDatabaseName") // set captured database - .tableList("yourDatabaseName.yourTableName") // set captured table - .username("yourUsername") - .password("yourPassword") - .deserializer(new JsonDebeziumDeserializationSchema()) // converts SourceRecord to JSON String - .build(); - - StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); - - // enable checkpoint - env.enableCheckpointing(3000); - - env - .fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source") - // set 4 parallel source tasks - .setParallelism(4) - .print().setParallelism(1); // use parallelism 1 for sink to keep message ordering - - env.execute("Print MySQL Snapshot + Binlog"); - } + public static void main(String[] args) throws Exception { + MySqlSource mySqlSource = MySqlSource.builder() + .hostname("yourHostname") + .port(yourPort) + .databaseList("yourDatabaseName") // set captured database + .tableList("yourDatabaseName.yourTableName") // set captured table + .username("yourUsername") + .password("yourPassword") + .deserializer(new JsonDebeziumDeserializationSchema()) // converts SourceRecord to JSON String + .build(); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + // enable checkpoint + env.enableCheckpointing(3000); + + env + .fromSource(mySqlSource, WatermarkStrategy.noWatermarks(), "MySQL Source") + // set 4 parallel source tasks + .setParallelism(4) + .print().setParallelism(1); // use parallelism 1 for sink to keep message ordering + + env.execute("Print MySQL Snapshot + Binlog"); + } } ``` + ### Deserialization + The following JSON data show the change event in JSON format. ```json @@ -135,16 +143,24 @@ The following JSON data show the change event in JSON format. "description": "Big 2-wheel scooter", "weight": 5.15 }, - "source": {...}, - "op": "u", // the operation type, "u" means this this is an update event - "ts_ms": 1589362330904, // the time at which the connector processed the event + "source": { + ... + }, + "op": "u", + // the operation type, "u" means this this is an update event + "ts_ms": 1589362330904, + // the time at which the connector processed the event "transaction": null } ``` -**Note:** Please refer [Debezium documentation](https://debezium.io/documentation/reference/1.6/connectors/mysql.html#mysql-events + +**Note:** Please +refer [Debezium documentation](https://debezium.io/documentation/reference/1.6/connectors/mysql.html#mysql-events ) to know the meaning of each field. -In some cases, users can use the `JsonDebeziumDeserializationSchema(true)` Constructor to enabled include schema in the message. Then the Debezium JSON message may look like this: +In some cases, users can use the `JsonDebeziumDeserializationSchema(true)` Constructor to enabled include schema in the +message. Then the Debezium JSON message may look like this: + ```json { "schema": { @@ -210,7 +226,9 @@ In some cases, users can use the `JsonDebeziumDeserializationSchema(true)` Const }, { "type": "struct", - "fields": {...}, + "fields": { + ... + }, "optional": false, "name": "io.debezium.connector.mysql.Source", "field": "source" @@ -242,28 +260,36 @@ In some cases, users can use the `JsonDebeziumDeserializationSchema(true)` Const "description": "Big 2-wheel scooter", "weight": 5.15 }, - "source": {...}, - "op": "u", // the operation type, "u" means this this is an update event - "ts_ms": 1589362330904, // the time at which the connector processed the event + "source": { + ... + }, + "op": "u", + // the operation type, "u" means this this is an update event + "ts_ms": 1589362330904, + // the time at which the connector processed the event "transaction": null } } ``` -Usually, it is recommended to exclude schema because schema fields makes the messages very verbose which reduces parsing performance. -The `JsonDebeziumDeserializationSchema` can also accept custom configuration of `JsonConverter`, for example if you want to obtain numeric output for decimal data, +Usually, it is recommended to exclude schema because schema fields makes the messages very verbose which reduces parsing +performance. + +The `JsonDebeziumDeserializationSchema` can also accept custom configuration of `JsonConverter`, for example if you want +to obtain numeric output for decimal data, you can construct `JsonDebeziumDeserializationSchema` as following: ```java - Map customConverterConfigs = new HashMap<>(); - customConverterConfigs.put(JsonConverterConfig.DECIMAL_FORMAT_CONFIG, "numeric"); - JsonDebeziumDeserializationSchema schema = - new JsonDebeziumDeserializationSchema(true, customConverterConfigs); + Map customConverterConfigs=new HashMap<>(); + customConverterConfigs.put(JsonConverterConfig.DECIMAL_FORMAT_CONFIG,"numeric"); + JsonDebeziumDeserializationSchema schema= + new JsonDebeziumDeserializationSchema(true,customConverterConfigs); ``` ## Building from source Prerequisites: + - git - Maven - At least Java 8 @@ -278,4 +304,5 @@ The dependencies are now available in your local `.m2` repository. ## License -The code in this repository is licensed under the [Apache Software License 2](https://github.com/ververica/flink-cdc-connectors/blob/master/LICENSE). +The code in this repository is licensed under +the [Apache Software License 2](https://github.com/ververica/flink-cdc-connectors/blob/master/LICENSE). diff --git a/docs/content/connectors/index.md b/docs/content/connectors/index.md index 5b4d0b34588..8c44cd383af 100644 --- a/docs/content/connectors/index.md +++ b/docs/content/connectors/index.md @@ -13,4 +13,5 @@ postgres-cdc sqlserver-cdc tidb-cdc db2-cdc +vitess-cdc ``` diff --git a/docs/content/connectors/vitess-cdc.md b/docs/content/connectors/vitess-cdc.md new file mode 100644 index 00000000000..4e9277c81c9 --- /dev/null +++ b/docs/content/connectors/vitess-cdc.md @@ -0,0 +1,309 @@ +# Vitess CDC Connector + +The Vitess CDC connector allows for reading of incremental data from Vitess cluster. The connector does not support snapshot feature at the moment. This document describes how to setup the Vitess CDC connector to run SQL queries against Vitess databases. +[Vitess debezium documentation](https://debezium.io/documentation/reference/connectors/vitess.html) + +Dependencies +------------ + +In order to setup the Vitess CDC connector, the following table provides dependency information for both projects using a build automation tool (such as Maven or SBT) and SQL Client with SQL JAR bundles. + +### Maven dependency + +``` + + com.ververica + flink-connector-vitess-cdc + 2.0.0 + +``` + +### SQL Client JAR + +Download [flink-sql-connector-vitess-cdc-2.0.0.jar](https://repo1.maven.org/maven2/com/ververica/flink-sql-connector-vitess-cdc/2.0.0/flink-sql-connector-vitess-cdc-2.0.0.jar) and put it under `/lib/`. + +Setup Vitess server +---------------- + +You can follow the Local Install via [Docker guide](https://vitess.io/docs/get-started/local-docker/), or the Vitess Operator for [Kubernetes guide](https://vitess.io/docs/get-started/operator/) to install Vitess. No special setup is needed to support Vitess connector. + +### Checklist +* Make sure that the VTGate host and its gRPC port (default is 15991) is accessible from the machine where the Vitess connector is installed +* Make sure that the VTCtld host and its gRPC port (default is 15999) is accessible from the machine where the Vitess connector is installed + +### gRPC authentication +Because Vitess connector reads change events from the VTGate VStream gRPC server, it does not need to connect directly to MySQL instances. +Therefore, no special database user and permissions are needed. At the moment, Vitess connector only supports unauthenticated access to the VTGate gRPC server. + +How to create a Vitess CDC table +---------------- + +The Vitess CDC table can be defined as following: + +```sql +-- checkpoint every 3000 milliseconds +Flink SQL> SET 'execution.checkpointing.interval' = '3s'; + +-- register a Vitess table 'orders' in Flink SQL +Flink SQL> CREATE TABLE orders ( + order_id INT, + order_date TIMESTAMP(0), + customer_name STRING, + price DECIMAL(10, 5), + product_id INT, + order_status BOOLEAN, + PRIMARY KEY(order_id) NOT ENFORCED + ) WITH ( + 'connector' = 'vitess-cdc', + 'hostname' = 'localhost', + 'port' = '3306', + 'keyspace' = 'mydb', + 'vtctl.hostname' = 'localhost', + 'table-name' = 'orders'); + +-- read snapshot and binlogs from orders table +Flink SQL> SELECT * FROM orders; +``` + +Connector Options +---------------- + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    OptionRequiredDefaultTypeDescription
    connectorrequired(none)StringSpecify what connector to use, here should be ‘vitess-cdc’.
    hostnamerequired(none)StringIP address or hostname of the Vitess database server (VTGate).
    keyspacerequired(none)  StringThe name of the keyspace from which to stream the changes.
    usernameoptional(none)StringAn optional username of the Vitess database server (VTGate). If not configured, unauthenticated VTGate gRPC is used.
    passwordoptional(none)StringAn optional password of the Vitess database server (VTGate). If not configured, unauthenticated VTGate gRPC is used.
    table-namerequired(none)StringTable name of the MySQL database to monitor.
    portoptional15991IntegerInteger port number of the VTCtld server.
    vtctld.hostrequired(none)String IP address or hostname of the VTCtld server.
    vtctld.portoptional15999Integer Integer port number of the VTCtld server.
    vtctld.useroptional(none) String An optional username of the VTCtld server. If not configured, unauthenticated VTCtld gRPC is used.
    vtctld.passwordoptional(none) String An optional password of the VTCtld server. If not configured, unauthenticated VTCtld gRPC is used.
    tablet.typeoptionalRDONLY String The type of Tablet (hence MySQL) from which to stream the changes: MASTER represents streaming from the master MySQL instance REPLICA represents streaming from the replica slave MySQL instance RDONLY represents streaming from the read-only slave MySQL instance.
    +
    + +Features +-------- + +### Incremental Reading + +The Vitess connector spends all its time streaming changes from the VTGate’s VStream gRPC service to which it is subscribed. The client receives changes from VStream as they are committed in the underlying MySQL server’s binlog at certain positions, which are referred to as VGTID. + +The VGTID in Vitess is the equivalent of GTID in MySQL, it describes the position in the VStream in which a change event happens. Typically, A VGTID has multiple shard GTIDs, each shard GTID is a tuple of (Keyspace, Shard, GTID), which describes the GTID position of a given shard. + +When subscribing to a VStream service, the connector needs to provide a VGTID and a Tablet Type (e.g. MASTER, REPLICA). The VGTID describes the position from which VStream should starts sending change events; the Tablet type describes which underlying MySQL instance (master or replica) in each shard do we read change events from. + +The first time the connector connects to a Vitess cluster, it gets the current VGTID from a Vitess component called VTCtld and provides the current VGTID to VStream. + +The Debezium Vitess connector acts as a gRPC client of VStream. When the connector receives changes it transforms the events into Debezium create, update, or delete events that include the VGTID of the event. The Vitess connector forwards these change events in records to the Kafka Connect framework, which is running in the same process. The Kafka Connect process asynchronously writes the change event records in the same order in which they were generated to the appropriate Kafka topic. + +#### Checkpoint + +Incremental snapshot reading provides the ability to perform checkpoint in chunk level. It resolves the checkpoint timeout problem in previous version with old snapshot reading mechanism. + +### Exactly-Once Processing + +The Vitess CDC connector is a Flink Source connector which will read table snapshot chunks first and then continues to read binlog, +both snapshot phase and binlog phase, Vitess CDC connector read with **exactly-once processing** even failures happen. + +### DataStream Source + +The Incremental Reading feature of Vitess CDC Source only exposes in SQL currently, if you're using DataStream, please use Vitess Source: + +```java +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.source.SourceFunction; +import com.ververica.cdc.debezium.JsonDebeziumDeserializationSchema; +import com.ververica.cdc.connectors.vitess.VitessSource; + +public class VitessSourceExample { + public static void main(String[] args) throws Exception { + SourceFunction sourceFunction = VitessSource.builder() + .hostname("localhost") + .port(15991) + .keyspace("inventory") + .username("flinkuser") + .password("flinkpw") + .vtctldConfig(VtctldConfig + .builder() + .hostname("localhost") + .port(15999) + .build()) + .deserializer(new JsonDebeziumDeserializationSchema()) // converts SourceRecord to JSON String + .build(); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + + env + .addSource(sourceFunction) + .print().setParallelism(1); // use parallelism 1 for sink to keep message ordering + + env.execute(); + } +} +``` + +**Note:** Please refer [Deserialization](../about.html#deserialization) for more details about the JSON deserialization. + +Data Type Mapping +---------------- + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    MySQL typeFlink SQL type
    TINYINTTINYINT
    + SMALLINT
    + TINYINT UNSIGNED    		SMALLINT
    + INT
    + MEDIUMINT
    + SMALLINT UNSIGNED    		INT
    + BIGINT
    + INT UNSIGNED    		BIGINT
    BIGINT UNSIGNEDDECIMAL(20, 0)
    BIGINTBIGINT
    FLOATFLOAT
    + DOUBLE
    + DOUBLE PRECISION    		DOUBLE
    + NUMERIC(p, s)
    + DECIMAL(p, s)    		DECIMAL(p, s)
    + BOOLEAN
    + TINYINT(1)    		BOOLEAN
    + CHAR(n)
    + VARCHAR(n)
    + TEXT    		STRING
    +
    diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index 7defc415c4c..77621861998 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -67,6 +67,14 @@ under the License. test + + mysql + mysql-connector-java + 8.0.26 + test + + + @@ -130,7 +138,30 @@ under the License. test - + + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + + org.testcontainers + jdbc + ${testcontainers.version} + test + + + + com.jayway.jsonpath + json-path + 2.4.0 + test + + + org.apache.logging.log4j @@ -138,7 +169,6 @@ under the License. ${log4j.version} test - @@ -157,4 +187,4 @@ under the License. - \ No newline at end of file + diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java index 15b0bc04eb8..49c04baba77 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java @@ -50,7 +50,7 @@ public static class Builder { private String username; private String password; private VtctldConfig vtctldConfig; - private TabletType tabletType = TabletType.REPLICA; + private TabletType tabletType = TabletType.RDONLY; private String[] tableIncludeList; private String[] tableExcludeList; private String[] columnIncludeList; @@ -200,13 +200,18 @@ public DebeziumSourceFunction build() { props.setProperty("database.server.name", "vitess_cdc_source"); props.setProperty("database.hostname", checkNotNull(hostname)); props.setProperty("database.port", String.valueOf(port)); - props.setProperty("database.user", checkNotNull(username)); - props.setProperty("database.password", checkNotNull(password)); props.setProperty("vitess.keyspace", checkNotNull(keyspace)); props.setProperty("vitess.tablet.type", tabletType.name()); props.setProperty("vitess.vtctld.host", checkNotNull(vtctldConfig.getHostname())); props.setProperty("vitess.vtctld.port", String.valueOf(vtctldConfig.getPort())); + if (username != null) { + props.setProperty("user", username); + } + if (vtctldConfig.getPassword() != null) { + props.setProperty("password", password); + } + if (vtctldConfig.getUsername() != null) { props.setProperty("vitess.vtctld.user", vtctldConfig.getUsername()); } diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java index a9808d6415b..feab2fb9b77 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/config/VtctldConfig.java @@ -18,6 +18,8 @@ package com.ververica.cdc.connectors.vitess.config; +import java.util.Objects; + /** VTCtld server configuration options. */ public class VtctldConfig { @@ -92,4 +94,41 @@ public VtctldConfig build() { return vtctldConfig; } } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + VtctldConfig that = (VtctldConfig) o; + return port == that.port + && Objects.equals(hostname, that.hostname) + && Objects.equals(username, that.username) + && Objects.equals(password, that.password); + } + + @Override + public int hashCode() { + return Objects.hash(hostname, port, username, password); + } + + @Override + public String toString() { + return "VtctldConfig{" + + "hostname='" + + hostname + + '\'' + + ", port=" + + port + + ", username='" + + username + + '\'' + + ", password='" + + password + + '\'' + + '}'; + } } diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java index ae5af87e18a..c4ad9f2e296 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java @@ -100,7 +100,7 @@ public class VitessTableFactory implements DynamicTableSourceFactory { private static final ConfigOption TABLET_TYPE = ConfigOptions.key("tablet-type") .stringType() - .defaultValue(TabletType.REPLICA.name()) + .defaultValue(TabletType.RDONLY.name()) .withDescription( "The type of Tablet (hence MySQL) from which to stream the changes:"); @@ -177,8 +177,6 @@ public Set> requiredOptions() { Set> options = new HashSet<>(); options.add(HOSTNAME); options.add(KEYSPACE); - options.add(USERNAME); - options.add(PASSWORD); options.add(VTCTL_HOSTNAME); options.add(TABLE_NAME); return options; @@ -189,6 +187,8 @@ public Set> optionalOptions() { Set> options = new HashSet<>(); options.add(PORT); options.add(VTCTL_PORT); + options.add(USERNAME); + options.add(PASSWORD); options.add(TABLET_TYPE); options.add(DECODING_PLUGIN_NAME); options.add(SLOT_NAME); diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java index b20f4d495b5..2001308d972 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java @@ -48,9 +48,9 @@ public class VitessTableSource implements ScanTableSource { private final TableSchema physicalSchema; - private String pluginName = "decoderbufs"; - private String slotName = "flink"; - private int port = 15991; // default 15991 port + private String pluginName; + private String slotName; + private int port; private String hostname; private String keyspace; private String username; @@ -78,8 +78,10 @@ public VitessTableSource( this.hostname = checkNotNull(hostname); this.keyspace = checkNotNull(keyspace); this.tableName = checkNotNull(tableName); - this.username = checkNotNull(username); - this.password = checkNotNull(password); + this.username = username; + this.password = password; + this.vtctldConfig = checkNotNull(vtctldConfig); + this.tabletType = checkNotNull(tabletType); this.pluginName = checkNotNull(pluginName); this.slotName = slotName; this.dbzProperties = dbzProperties; @@ -179,6 +181,43 @@ public int hashCode() { dbzProperties); } + @Override + public String toString() { + return "VitessTableSource{" + + "physicalSchema=" + + physicalSchema + + ", pluginName='" + + pluginName + + '\'' + + ", slotName='" + + slotName + + '\'' + + ", port=" + + port + + ", hostname='" + + hostname + + '\'' + + ", keyspace='" + + keyspace + + '\'' + + ", username='" + + username + + '\'' + + ", password='" + + password + + '\'' + + ", tableName='" + + tableName + + '\'' + + ", vtctldConfig=" + + vtctldConfig + + ", tabletType=" + + tabletType + + ", dbzProperties=" + + dbzProperties + + '}'; + } + @Override public String asSummaryString() { return "Vitess-CDC"; diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java new file mode 100644 index 00000000000..bfacd0220a3 --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java @@ -0,0 +1,310 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.vervetica.cdc.connectors.vitess; + +import org.apache.flink.api.common.state.BroadcastState; +import org.apache.flink.api.common.state.KeyedStateStore; +import org.apache.flink.api.common.state.ListState; +import org.apache.flink.api.common.state.ListStateDescriptor; +import org.apache.flink.api.common.state.MapStateDescriptor; +import org.apache.flink.api.common.state.OperatorStateStore; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.core.testutils.CheckedThread; +import org.apache.flink.runtime.state.FunctionInitializationContext; +import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; +import org.apache.flink.streaming.util.MockStreamingRuntimeContext; +import org.apache.flink.util.Collector; + +import com.ververica.cdc.connectors.utils.TestSourceContext; +import com.ververica.cdc.connectors.vitess.VitessSource; +import com.ververica.cdc.connectors.vitess.config.TabletType; +import com.ververica.cdc.connectors.vitess.config.VtctldConfig; +import com.ververica.cdc.debezium.DebeziumDeserializationSchema; +import com.ververica.cdc.debezium.DebeziumSourceFunction; +import org.apache.kafka.connect.source.SourceRecord; +import org.junit.Before; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.ververica.cdc.connectors.utils.AssertUtils.assertDelete; +import static com.ververica.cdc.connectors.utils.AssertUtils.assertInsert; +import static com.ververica.cdc.connectors.utils.AssertUtils.assertUpdate; + +/** Tests for {@link VitessSource} which also heavily tests {@link DebeziumSourceFunction}. */ +public class VitessSourceTest extends VitessTestBase { + + @Before + public void before() { + initializeMysqlTable("inventory"); + } + + @Test + public void testConsumingAllEvents() throws Exception { + DebeziumSourceFunction source = createVitessSqlSource(0); + TestSourceContext sourceContext = new TestSourceContext<>(); + + setupSource(source); + + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + + // start the source + final CheckedThread runThread = + new CheckedThread() { + @Override + public void go() throws Exception { + source.run(sourceContext); + } + }; + runThread.start(); + + waitForSourceToStart(Duration.ofSeconds(60), source); + List records; + + statement.execute( + "INSERT INTO test.products VALUES (default,'robot','Toy robot',1.304)"); // 110 + records = drain(sourceContext, 1); + assertInsert(records.get(0), "id", 101); + + statement.execute( + "INSERT INTO test.products VALUES (1001,'roy','old robot',1234.56)"); // 1001 + records = drain(sourceContext, 1); + assertInsert(records.get(0), "id", 1001); + + // --------------------------------------------------------------------------------------------------------------- + // Changing the primary key of a row should result in 2 events: INSERT, DELETE + // (TOMBSTONE is dropped) + // --------------------------------------------------------------------------------------------------------------- + statement.execute( + "UPDATE test.products SET id=2001, description='really old robot' WHERE id=1001"); + records = drain(sourceContext, 2); + assertDelete(records.get(0), "id", 1001); + assertInsert(records.get(1), "id", 2001); + + // --------------------------------------------------------------------------------------------------------------- + // Simple UPDATE (with no schema changes) + // --------------------------------------------------------------------------------------------------------------- + statement.execute("UPDATE test.products SET weight=1345.67 WHERE id=2001"); + records = drain(sourceContext, 1); + assertUpdate(records.get(0), "id", 2001); + + // --------------------------------------------------------------------------------------------------------------- + // Change our schema with a fully-qualified name; we should still see this event + // --------------------------------------------------------------------------------------------------------------- + // Add a column with default to the 'products' table and explicitly update one record + // ... + statement.execute( + "ALTER TABLE test.products ADD COLUMN volume FLOAT, ADD COLUMN alias VARCHAR(30) NULL"); + statement.execute("UPDATE test.products SET volume=13.5 WHERE id=2001"); + records = drain(sourceContext, 1); + assertUpdate(records.get(0), "id", 2001); + + // cleanup + source.cancel(); + source.close(); + runThread.sync(); + } + } + + // ------------------------------------------------------------------------------------------ + // Utilities + // ------------------------------------------------------------------------------------------ + + private DebeziumSourceFunction createVitessSqlSource(int heartbeatInterval) { + Properties properties = new Properties(); + properties.setProperty("heartbeat.interval.ms", String.valueOf(heartbeatInterval)); + return VitessSource.builder() + .hostname(VITESS_CONTAINER.getHost()) + .port(VITESS_CONTAINER.getGrpcPort()) + .keyspace(VITESS_CONTAINER.getKeyspace()) + .tabletType(TabletType.MASTER) + .tableIncludeList("test.products") + .vtctldConfig( + VtctldConfig.builder() + .hostname(VITESS_CONTAINER.getHost()) + .port(VITESS_CONTAINER.getVtctldGrpcPort()) + .build()) + .deserializer(new ForwardDeserializeSchema()) + .debeziumProperties(properties) + .build(); + } + + private List drain(TestSourceContext sourceContext, int expectedRecordCount) + throws Exception { + List allRecords = new ArrayList<>(); + LinkedBlockingQueue> queue = sourceContext.getCollectedOutputs(); + while (allRecords.size() < expectedRecordCount) { + StreamRecord record = queue.poll(1000, TimeUnit.SECONDS); + if (record != null) { + allRecords.add(record.getValue()); + } else { + throw new RuntimeException( + "Can't receive " + expectedRecordCount + " elements before timeout."); + } + } + + return allRecords; + } + + private boolean waitForSourceToStart( + Duration timeout, DebeziumSourceFunction source) + throws InterruptedException { + long now = System.currentTimeMillis(); + long stop = now + timeout.toMillis(); + while (System.currentTimeMillis() < stop) { + if (source.getDebeziumStarted()) { + break; + } + Thread.sleep(10); // save CPU + } + Thread.sleep(10000); // Wait for full start + return source.getDebeziumStarted(); + } + + private static void setupSource(DebeziumSourceFunction source) throws Exception { + setupSource( + source, false, null, null, + true, // enable checkpointing; auto commit should be ignored + 0, 1); + } + + private static void setupSource( + DebeziumSourceFunction source, + boolean isRestored, + ListState restoredOffsetState, + ListState restoredHistoryState, + boolean isCheckpointingEnabled, + int subtaskIndex, + int totalNumSubtasks) + throws Exception { + + // run setup procedure in operator life cycle + source.setRuntimeContext( + new MockStreamingRuntimeContext( + isCheckpointingEnabled, totalNumSubtasks, subtaskIndex)); + source.initializeState( + new MockFunctionInitializationContext( + isRestored, + new MockOperatorStateStore(restoredOffsetState, restoredHistoryState))); + source.open(new Configuration()); + } + + private static class ForwardDeserializeSchema + implements DebeziumDeserializationSchema { + + private static final long serialVersionUID = 2975058057832211228L; + + @Override + public void deserialize(SourceRecord record, Collector out) throws Exception { + out.collect(record); + } + + @Override + public TypeInformation getProducedType() { + return TypeInformation.of(SourceRecord.class); + } + } + + private static class MockOperatorStateStore implements OperatorStateStore { + + private final ListState restoredOffsetListState; + private final ListState restoredHistoryListState; + + private MockOperatorStateStore( + ListState restoredOffsetListState, ListState restoredHistoryListState) { + this.restoredOffsetListState = restoredOffsetListState; + this.restoredHistoryListState = restoredHistoryListState; + } + + @Override + @SuppressWarnings("unchecked") + public ListState getUnionListState(ListStateDescriptor stateDescriptor) + throws Exception { + if (stateDescriptor.getName().equals(DebeziumSourceFunction.OFFSETS_STATE_NAME)) { + return (ListState) restoredOffsetListState; + } else if (stateDescriptor + .getName() + .equals(DebeziumSourceFunction.HISTORY_RECORDS_STATE_NAME)) { + return (ListState) restoredHistoryListState; + } else { + throw new IllegalStateException("Unknown state."); + } + } + + @Override + public BroadcastState getBroadcastState( + MapStateDescriptor stateDescriptor) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public ListState getListState(ListStateDescriptor stateDescriptor) + throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public Set getRegisteredStateNames() { + throw new UnsupportedOperationException(); + } + + @Override + public Set getRegisteredBroadcastStateNames() { + throw new UnsupportedOperationException(); + } + } + + private static class MockFunctionInitializationContext + implements FunctionInitializationContext { + + private final boolean isRestored; + private final OperatorStateStore operatorStateStore; + + private MockFunctionInitializationContext( + boolean isRestored, OperatorStateStore operatorStateStore) { + this.isRestored = isRestored; + this.operatorStateStore = operatorStateStore; + } + + @Override + public boolean isRestored() { + return isRestored; + } + + @Override + public OperatorStateStore getOperatorStateStore() { + return operatorStateStore; + } + + @Override + public KeyedStateStore getKeyedStateStore() { + throw new UnsupportedOperationException(); + } + } +} diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java new file mode 100644 index 00000000000..d3958e549c2 --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.vervetica.cdc.connectors.vitess; + +import org.apache.flink.test.util.AbstractTestBase; + +import com.vervetica.cdc.connectors.vitess.container.VitessContainer; +import org.junit.BeforeClass; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.vervetica.cdc.connectors.vitess.container.VitessContainer.GRPC_PORT; +import static com.vervetica.cdc.connectors.vitess.container.VitessContainer.MYSQL_PORT; +import static com.vervetica.cdc.connectors.vitess.container.VitessContainer.VTCTLD_GRPC_PORT; +import static org.junit.Assert.assertNotNull; + +/** Basic class for testing Vitess source, this contains a Vitess container. */ +public abstract class VitessTestBase extends AbstractTestBase { + + private static final Logger LOG = LoggerFactory.getLogger(VitessTestBase.class); + private static final Pattern COMMENT_PATTERN = Pattern.compile("^(.*)--.*$"); + + protected static final VitessContainer VITESS_CONTAINER = + (VitessContainer) + new VitessContainer() + .withKeyspace("test") + .withUsername("flinkuser") + .withPassword("flinkpwd") + .withExposedPorts(MYSQL_PORT, GRPC_PORT, VTCTLD_GRPC_PORT) + .withLogConsumer(new Slf4jLogConsumer(LOG)); + + @BeforeClass + public static void startContainers() { + LOG.info("Starting containers..."); + Startables.deepStart(Stream.of(VITESS_CONTAINER)).join(); + LOG.info("Containers are started."); + } + + public Connection getJdbcConnection() throws SQLException { + return DriverManager.getConnection(VITESS_CONTAINER.getJdbcUrl()); + } + + /** + * Executes a JDBC statement using the default jdbc config without autocommitting the + * connection. + */ + protected void initializeMysqlTable(String sqlFile) { + final String ddlFile = String.format("ddl/%s.sql", sqlFile); + final URL ddlTestFile = VitessTestBase.class.getClassLoader().getResource(ddlFile); + assertNotNull("Cannot locate " + ddlFile, ddlTestFile); + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + final List statements = + Arrays.stream( + Files.readAllLines(Paths.get(ddlTestFile.toURI())).stream() + .map(String::trim) + .filter(x -> !x.startsWith("--") && !x.isEmpty()) + .map( + x -> { + final Matcher m = + COMMENT_PATTERN.matcher(x); + return m.matches() ? m.group(1) : x; + }) + .collect(Collectors.joining("\n")) + .split(";")) + .collect(Collectors.toList()); + for (String stmt : statements) { + statement.execute(stmt); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java new file mode 100644 index 00000000000..ddf8de494ec --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java @@ -0,0 +1,103 @@ +package com.vervetica.cdc.connectors.vitess.container; + +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** Vitess container. */ +public class VitessContainer extends JdbcDatabaseContainer { + + public static final String IMAGE = "vitess/vttestserver"; + public static final String DEFAULT_TAG = "mysql80"; + private static final Integer VITESS_PORT = 15991; + public static final Integer GRPC_PORT = VITESS_PORT + 1; + public static final Integer VTCTLD_GRPC_PORT = VITESS_PORT + 8; + public static final Integer MYSQL_PORT = VITESS_PORT + 3; + + private String keyspaces = "test"; + private String username = "flinkuser"; + private String password = "flinkpwd"; + + public VitessContainer() { + this(DEFAULT_TAG); + } + + public VitessContainer(String tag) { + super(IMAGE + ":" + tag); + } + + @Override + protected void configure() { + addEnv("PORT", VITESS_PORT.toString()); + addEnv("KEYSPACES", getKeyspace()); + addEnv("NUM_SHARDS", "1"); + addEnv("MYSQL_BIND_HOST", "0.0.0.0"); + } + + @Override + public String getDriverClassName() { + try { + Class.forName("com.mysql.cj.jdbc.Driver"); + return "com.mysql.cj.jdbc.Driver"; + } catch (ClassNotFoundException e) { + return "com.mysql.jdbc.Driver"; + } + } + + @Override + public String getJdbcUrl() { + return "jdbc:mysql://" + getHost() + ":" + getMysqlPort() + "/" + getKeyspace(); + } + + @Override + public String getUsername() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + public String getKeyspace() { + return keyspaces; + } + + public Integer getMysqlPort() { + return this.getMappedPort(MYSQL_PORT); + } + + public Integer getGrpcPort() { + return this.getMappedPort(GRPC_PORT); + } + + public Integer getVtctldGrpcPort() { + return this.getMappedPort(VTCTLD_GRPC_PORT); + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + + @Override + public VitessContainer withDatabaseName(final String keyspace) { + this.keyspaces = keyspace; + return this; + } + + public VitessContainer withKeyspace(String keyspace) { + this.keyspaces = keyspace; + return this; + } + + @Override + public VitessContainer withUsername(final String username) { + this.username = username; + return this; + } + + @Override + public VitessContainer withPassword(final String password) { + this.password = password; + return this; + } +} diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java new file mode 100644 index 00000000000..bf97a89a82b --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.vervetica.cdc.connectors.vitess.table; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.EnvironmentSettings; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.planner.factories.TestValuesTableFactory; +import org.apache.flink.table.utils.LegacyRowResource; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import com.vervetica.cdc.connectors.vitess.VitessTestBase; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** Integration tests for MySQL binlog SQL source. */ +public class VitessConnectorITCase extends VitessTestBase { + + private final StreamExecutionEnvironment env = + StreamExecutionEnvironment.getExecutionEnvironment(); + private final StreamTableEnvironment tEnv = + StreamTableEnvironment.create( + env, + EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build()); + + @ClassRule public static LegacyRowResource usesLegacyRows = LegacyRowResource.INSTANCE; + + @Before + public void before() { + TestValuesTableFactory.clearAllData(); + env.setParallelism(1); + } + + @Test + public void testConsumingAllEvents() + throws SQLException, ExecutionException, InterruptedException { + initializeMysqlTable("inventory"); + String sourceDDL = + String.format( + "CREATE TABLE debezium_source (" + + " `id` INT NOT NULL," + + " name STRING," + + " description STRING," + + " weight DECIMAL(10,3)," + + " primary key (`id`) not enforced" + + ") WITH (" + + " 'connector' = 'vitess-cdc'," + + " 'tablet-type' = 'MASTER'," + + " 'hostname' = '%s'," + + " 'port' = '%s'," + + " 'vtctl.hostname' = '%s'," + + " 'vtctl.port' = '%s'," + + " 'keyspace' = '%s'," + + " 'table-name' = '%s'" + + ")", + VITESS_CONTAINER.getHost(), + VITESS_CONTAINER.getGrpcPort(), + VITESS_CONTAINER.getHost(), + VITESS_CONTAINER.getVtctldGrpcPort(), + VITESS_CONTAINER.getKeyspace(), + "test.products"); + String sinkDDL = + "CREATE TABLE sink (" + + " name STRING," + + " weightSum DECIMAL(10,3)," + + " PRIMARY KEY (name) NOT ENFORCED" + + ") WITH (" + + " 'connector' = 'values'," + + " 'sink-insert-only' = 'false'," + + " 'sink-expected-messages-num' = '20'" + + ")"; + tEnv.executeSql(sourceDDL); + tEnv.executeSql(sinkDDL); + + // async submit job + TableResult result = + tEnv.executeSql( + "INSERT INTO sink SELECT name, SUM(weight) FROM debezium_source GROUP BY name"); + + // Vitess source doesn't read snapshot data. Source will be empty at first. + // There's no way knowing if it's started, using sleep here. + Thread.sleep(10000); + + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "INSERT INTO test.products \n" + + "VALUES (default,'scooter','Small 2-wheel scooter',3.14),\n" + + " (default,'car battery','12V car battery',8.1),\n" + + " (default,'12-pack drill bits','12-pack of drill bits with sizes ranging from #40 to #3',0.8),\n" + + " (default,'hammer','12oz carpenters hammer',0.75),\n" + + " (default,'hammer','14oz carpenters hammer',0.875),\n" + + " (default,'hammer','16oz carpenters hammer',1.0),\n" + + " (default,'rocks','box of assorted rocks',5.3),\n" + + " (default,'jacket','water resistent black wind breaker',0.1),\n" + + " (default,'spare tire','24 inch spare tire',22.2);"); + statement.execute( + "UPDATE test.products SET description='18oz carpenter hammer' WHERE id=106;"); + statement.execute("UPDATE test.products SET weight='5.1' WHERE id=107;"); + statement.execute( + "INSERT INTO test.products VALUES (default,'jacket','water resistent white wind breaker',0.2);"); // 110 + statement.execute( + "INSERT INTO test.products VALUES (default,'scooter','Big 2-wheel scooter ',5.18);"); + statement.execute( + "UPDATE test.products SET description='new water resistent white wind breaker', weight='0.5' WHERE id=110;"); + statement.execute("UPDATE test.products SET weight='5.17' WHERE id=111;"); + statement.execute("DELETE FROM test.products WHERE id=111;"); + } + + waitForSinkSize("sink", 20); + + List expected = + Arrays.asList( + "scooter,3.140", + "car battery,8.100", + "12-pack drill bits,0.800", + "hammer,2.625", + "rocks,5.100", + "jacket,0.600", + "spare tire,22.200"); + + List actual = TestValuesTableFactory.getResults("sink"); + assertEqualsInAnyOrder(expected, actual); + result.getJobClient().get().cancel().get(); + } + + @Test // TODO add DATE, TIMESTAMP, TIME type mapping + public void testAllTypes() throws Throwable { + initializeMysqlTable("column_type_test"); + String sourceDDL = + String.format( + "CREATE TABLE full_types (\n" + + " `id` INT NOT NULL,\n" + + " tiny_c TINYINT,\n" + + " tiny_un_c SMALLINT ,\n" + + " small_c SMALLINT,\n" + + " small_un_c INT,\n" + + " int_c INT ,\n" + + " int_un_c BIGINT,\n" + + " int11_c BIGINT,\n" + + " big_c BIGINT,\n" + + " varchar_c STRING,\n" + + " char_c STRING,\n" + + " float_c FLOAT,\n" + + " double_c DOUBLE,\n" + + " decimal_c DECIMAL(8, 4),\n" + + " numeric_c DECIMAL(6, 0),\n" + + " boolean_c BOOLEAN,\n" + // + " date_c DATE,\n" + // + " time_c TIME(0),\n" + // + " datetime3_c TIMESTAMP(3),\n" + // + " datetime6_c TIMESTAMP(6),\n" + // + " timestamp_c TIMESTAMP(0),\n" + // + " file_uuid BYTES,\n" + + " primary key (`id`) not enforced" + + ") WITH (" + + " 'connector' = 'vitess-cdc'," + + " 'tablet-type' = 'MASTER'," + + " 'hostname' = '%s'," + + " 'port' = '%s'," + + " 'vtctl.hostname' = '%s'," + + " 'vtctl.port' = '%s'," + + " 'keyspace' = '%s'," + + " 'table-name' = '%s'" + + ")", + VITESS_CONTAINER.getHost(), + VITESS_CONTAINER.getGrpcPort(), + VITESS_CONTAINER.getHost(), + VITESS_CONTAINER.getVtctldGrpcPort(), + VITESS_CONTAINER.getKeyspace(), + "test.full_types"); + tEnv.executeSql(sourceDDL); + + // async submit job + TableResult result = tEnv.executeSql("SELECT * FROM full_types"); + + // Vitess source doesn't read snapshot data. Source will be empty at first. + // There's no way knowing if it's started, using sleep here. + Thread.sleep(10000); + + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "INSERT INTO test.full_types VALUES (\n" + + " DEFAULT, 127, 255, 32767, 65535, 2147483647, 4294967295, 2147483647, 9223372036854775807,\n" + + " 'Hello World', 'abc', 123.102, 404.4443, 123.4567, 345.6, true);"); + statement.execute("UPDATE test.full_types SET varchar_c = 'Bye World' WHERE id=1;"); + } + + waitForSnapshotStarted(result.collect()); + + List expected = + Arrays.asList( + "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Hello World,abc,123.102,404.4443,123.4567,346,true", + "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Hello World,abc,123.102,404.4443,123.4567,346,true", + "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Bye World,abc,123.102,404.4443,123.4567,346,true"); + + List actual = fetchRows(result.collect(), expected.size()); + assertEquals(expected, actual); + result.getJobClient().get().cancel().get(); + } + + private static List fetchRows(Iterator iter, int size) { + List rows = new ArrayList<>(size); + while (size > 0 && iter.hasNext()) { + Row row = iter.next(); + rows.add(row.toString()); + size--; + } + return rows; + } + + public static void assertEqualsInAnyOrder(List actual, List expected) { + assertTrue(actual != null && expected != null); + assertEquals( + actual.stream().sorted().collect(Collectors.toList()), + expected.stream().sorted().collect(Collectors.toList())); + } + + private static void waitForSnapshotStarted(CloseableIterator iterator) throws Exception { + while (!iterator.hasNext()) { + Thread.sleep(100); + } + } + + private static void waitForSinkSize(String sinkName, int expectedSize) + throws InterruptedException { + while (sinkSize(sinkName) < expectedSize) { + Thread.sleep(100); + } + } + + private static int sinkSize(String sinkName) { + synchronized (TestValuesTableFactory.class) { + try { + return TestValuesTableFactory.getRawResults(sinkName).size(); + } catch (IllegalArgumentException e) { + // job is not started yet + return 0; + } + } + } +} diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java new file mode 100644 index 00000000000..330576f6b32 --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.vervetica.cdc.connectors.vitess.table; + +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.CatalogTable; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ObjectIdentifier; +import org.apache.flink.table.catalog.ResolvedCatalogTable; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.catalog.UniqueConstraint; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.factories.Factory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.utils.TableSchemaUtils; +import org.apache.flink.util.ExceptionUtils; + +import com.ververica.cdc.connectors.vitess.config.TabletType; +import com.ververica.cdc.connectors.vitess.config.VtctldConfig; +import com.ververica.cdc.connectors.vitess.table.VitessTableFactory; +import com.ververica.cdc.connectors.vitess.table.VitessTableSource; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.apache.flink.table.api.TableSchema.fromResolvedSchema; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** Test for {@link VitessTableSource} created by {@link VitessTableFactory}. */ +public class VitessTableFactoryTest { + + private static final ResolvedSchema SCHEMA = + new ResolvedSchema( + Arrays.asList( + Column.physical("aaa", DataTypes.INT().notNull()), + Column.physical("bbb", DataTypes.STRING().notNull()), + Column.physical("ccc", DataTypes.DOUBLE()), + Column.physical("ddd", DataTypes.DECIMAL(31, 18)), + Column.physical("eee", DataTypes.TIMESTAMP(3))), + new ArrayList<>(), + UniqueConstraint.primaryKey("pk", Arrays.asList("bbb", "aaa"))); + + private static final String MY_LOCALHOST = "localhost"; + private static final String MY_USERNAME = "flinkuser"; + private static final String MY_PASSWORD = "flinkpw"; + private static final String MY_KEYSPACE = "myDB"; + private static final String MY_TABLE = "myTable"; + private static final Properties PROPERTIES = new Properties(); + + @Test + public void testCommonProperties() { + Map properties = getAllOptions(); + + // validation for source + DynamicTableSource actualSource = createTableSource(properties); + VitessTableSource expectedSource = + new VitessTableSource( + TableSchemaUtils.getPhysicalSchema(fromResolvedSchema(SCHEMA)), + 15991, + MY_LOCALHOST, + MY_KEYSPACE, + MY_TABLE, + null, + null, + VtctldConfig.builder().hostname(MY_LOCALHOST).port(15999).build(), + TabletType.RDONLY, + "decoderbufs", + "flink", + PROPERTIES); + assertEquals(expectedSource, actualSource); + } + + @Test + public void testOptionalProperties() { + Map options = getAllOptions(); + options.put("port", "5444"); + options.put("vtctl.port", "5445"); + options.put("decoding.plugin.name", "wal2json"); + options.put("debezium.snapshot.mode", "never"); + options.put("slot.name", "flink"); + options.put("tablet-type", "MASTER"); + options.put("username", MY_USERNAME); + options.put("password", MY_PASSWORD); + + DynamicTableSource actualSource = createTableSource(options); + Properties dbzProperties = new Properties(); + dbzProperties.put("snapshot.mode", "never"); + VitessTableSource expectedSource = + new VitessTableSource( + TableSchemaUtils.getPhysicalSchema(fromResolvedSchema(SCHEMA)), + 5444, + MY_LOCALHOST, + MY_KEYSPACE, + MY_TABLE, + MY_USERNAME, + MY_PASSWORD, + VtctldConfig.builder().hostname(MY_LOCALHOST).port(5445).build(), + TabletType.MASTER, + "wal2json", + "flink", + dbzProperties); + assertEquals(expectedSource, actualSource); + } + + @Test + public void testValidation() { + // validate illegal port + try { + Map properties = getAllOptions(); + properties.put("port", "123b"); + + createTableSource(properties); + fail("exception expected"); + } catch (Throwable t) { + assertTrue( + ExceptionUtils.findThrowableWithMessage( + t, "Could not parse value '123b' for key 'port'.") + .isPresent()); + } + + // validate missing required + Factory factory = new VitessTableFactory(); + for (ConfigOption requiredOption : factory.requiredOptions()) { + Map properties = getAllOptions(); + properties.remove(requiredOption.key()); + + try { + createTableSource(properties); + fail("exception expected"); + } catch (Throwable t) { + assertTrue( + ExceptionUtils.findThrowableWithMessage( + t, + "Missing required options are:\n\n" + requiredOption.key()) + .isPresent()); + } + } + + // validate unsupported option + try { + Map properties = getAllOptions(); + properties.put("unknown", "abc"); + + createTableSource(properties); + fail("exception expected"); + } catch (Throwable t) { + assertTrue( + ExceptionUtils.findThrowableWithMessage(t, "Unsupported options:\n\nunknown") + .isPresent()); + } + } + + private Map getAllOptions() { + Map options = new HashMap<>(); + options.put("connector", "vitess-cdc"); + options.put("hostname", MY_LOCALHOST); + options.put("keyspace", MY_KEYSPACE); + options.put("vtctl.hostname", MY_LOCALHOST); + options.put("table-name", MY_TABLE); + return options; + } + + private static DynamicTableSource createTableSource(Map options) { + return FactoryUtil.createTableSource( + null, + ObjectIdentifier.of("default", "default", "t1"), + new ResolvedCatalogTable( + CatalogTable.of( + fromResolvedSchema(SCHEMA).toSchema(), + "mock source", + new ArrayList<>(), + options), + SCHEMA), + new Configuration(), + VitessTableFactoryTest.class.getClassLoader(), + false); + } +} diff --git a/flink-connector-vitess-cdc/src/test/resources/ddl/column_type_test.sql b/flink-connector-vitess-cdc/src/test/resources/ddl/column_type_test.sql new file mode 100644 index 00000000000..7bf4fe191ee --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/resources/ddl/column_type_test.sql @@ -0,0 +1,44 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +USE test; +DROP TABLE IF EXISTS full_types; +-- TODO add DATE, DATETIME, TIMESTAMP, TIME type mapping +CREATE TABLE full_types ( + id INT AUTO_INCREMENT NOT NULL, + tiny_c TINYINT, + tiny_un_c TINYINT UNSIGNED, + small_c SMALLINT, + small_un_c SMALLINT UNSIGNED, + int_c INTEGER , + int_un_c INTEGER UNSIGNED, + int11_c INT(11) , + big_c BIGINT, + varchar_c VARCHAR(255), + char_c CHAR(3), + float_c FLOAT, + double_c DOUBLE, + decimal_c DECIMAL(8, 4), + numeric_c NUMERIC(6, 0), + boolean_c BOOLEAN, +-- date_c DATE, +-- time_c TIME(0), +-- datetime3_c DATETIME(3), +-- datetime6_c DATETIME(6), +-- timestamp_c TIMESTAMP, +-- file_uuid BINARY(16), + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; diff --git a/flink-connector-vitess-cdc/src/test/resources/ddl/inventory.sql b/flink-connector-vitess-cdc/src/test/resources/ddl/inventory.sql new file mode 100644 index 00000000000..b6be784ca82 --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/resources/ddl/inventory.sql @@ -0,0 +1,24 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +USE test; +DROP TABLE IF EXISTS products; +CREATE TABLE products ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL DEFAULT 'flink', + description VARCHAR(512), + weight FLOAT +); +ALTER TABLE products AUTO_INCREMENT = 101; diff --git a/flink-connector-vitess-cdc/src/test/resources/log4j2-test.properties b/flink-connector-vitess-cdc/src/test/resources/log4j2-test.properties new file mode 100644 index 00000000000..b82a9606dbf --- /dev/null +++ b/flink-connector-vitess-cdc/src/test/resources/log4j2-test.properties @@ -0,0 +1,28 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level=INFO +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%t] %-5p %c - %m%n From a4f041286ee8d43901423916f27d16c02213167f Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Thu, 23 Sep 2021 12:47:41 +0300 Subject: [PATCH 04/20] chore[vitess-cdc]: Add license --- .../vitess/container/VitessContainer.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java index ddf8de494ec..ef6990a250b 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/container/VitessContainer.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.vervetica.cdc.connectors.vitess.container; import org.testcontainers.containers.JdbcDatabaseContainer; From ff17650415bb301044897645c91b4ae732f44039 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Thu, 23 Sep 2021 15:51:03 +0300 Subject: [PATCH 05/20] tests[vitess-cdc]: Remove LegacyRowResource.INSTANCE usage --- .../vitess/table/VitessConnectorITCase.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java index bf97a89a82b..0324117b6e3 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java @@ -23,13 +23,11 @@ import org.apache.flink.table.api.TableResult; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import org.apache.flink.table.planner.factories.TestValuesTableFactory; -import org.apache.flink.table.utils.LegacyRowResource; import org.apache.flink.types.Row; import org.apache.flink.util.CloseableIterator; import com.vervetica.cdc.connectors.vitess.VitessTestBase; import org.junit.Before; -import org.junit.ClassRule; import org.junit.Test; import java.sql.Connection; @@ -55,8 +53,6 @@ public class VitessConnectorITCase extends VitessTestBase { env, EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build()); - @ClassRule public static LegacyRowResource usesLegacyRows = LegacyRowResource.INSTANCE; - @Before public void before() { TestValuesTableFactory.clearAllData(); @@ -143,13 +139,13 @@ public void testConsumingAllEvents() List expected = Arrays.asList( - "scooter,3.140", - "car battery,8.100", - "12-pack drill bits,0.800", - "hammer,2.625", - "rocks,5.100", - "jacket,0.600", - "spare tire,22.200"); + "+I[scooter, 3.140]", + "+I[car battery, 8.100]", + "+I[12-pack drill bits, 0.800]", + "+I[hammer, 2.625]", + "+I[rocks, 5.100]", + "+I[jacket, 0.600]", + "+I[spare tire, 22.200]"); List actual = TestValuesTableFactory.getResults("sink"); assertEqualsInAnyOrder(expected, actual); @@ -223,9 +219,9 @@ public void testAllTypes() throws Throwable { List expected = Arrays.asList( - "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Hello World,abc,123.102,404.4443,123.4567,346,true", - "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Hello World,abc,123.102,404.4443,123.4567,346,true", - "1,127,255,32767,65535,2147483647,4294967295,2147483647,9223372036854775807,Bye World,abc,123.102,404.4443,123.4567,346,true"); + "+I[1, 127, 255, 32767, 65535, 2147483647, 4294967295, 2147483647, 9223372036854775807, Hello World, abc, 123.102, 404.4443, 123.4567, 346, true]", + "-U[1, 127, 255, 32767, 65535, 2147483647, 4294967295, 2147483647, 9223372036854775807, Hello World, abc, 123.102, 404.4443, 123.4567, 346, true]", + "+U[1, 127, 255, 32767, 65535, 2147483647, 4294967295, 2147483647, 9223372036854775807, Bye World, abc, 123.102, 404.4443, 123.4567, 346, true]"); List actual = fetchRows(result.collect(), expected.size()); assertEquals(expected, actual); From 86989505f42f42728e4daee4d74fb84b44a59ec1 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Tue, 23 Nov 2021 14:00:54 +0200 Subject: [PATCH 06/20] fix[vitess-cdc]: Update flink-cdc-connectors for newer debezium version --- flink-connector-vitess-cdc/pom.xml | 2 +- .../cdc/connectors/vitess/table/VitessTableSource.java | 10 +++++++--- flink-sql-connector-vitess-cdc/pom.xml | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index 77621861998..29a3408ff8a 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.1-SNAPSHOT + 2.2-SNAPSHOT 4.0.0 diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java index 2001308d972..cc5e791a74a 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java @@ -99,12 +99,16 @@ public ChangelogMode getChangelogMode() { @Override public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { - RowType rowType = (RowType) physicalSchema.toRowDataType().getLogicalType(); + RowType physicalDataType = + (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType(); TypeInformation typeInfo = scanContext.createTypeInformation(physicalSchema.toRowDataType()); DebeziumDeserializationSchema deserializer = - new RowDataDebeziumDeserializeSchema( - rowType, typeInfo, ((rowData, rowKind) -> {}), ZoneId.of("UTC")); + RowDataDebeziumDeserializeSchema.newBuilder() + .setPhysicalRowType(physicalDataType) + .setResultTypeInfo(typeInfo) + .setServerTimeZone(ZoneId.of("UTC")) + .build(); DebeziumSourceFunction sourceFunction = VitessSource.builder() diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index e7176141aa4..25f4aed34e5 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.1-SNAPSHOT + 2.2-SNAPSHOT 4.0.0 @@ -75,4 +75,4 @@ under the License. - \ No newline at end of file + From 8a7b0a7a919d1665842eb022e7d64e3b57ac54ca Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Wed, 22 Dec 2021 11:55:17 +0200 Subject: [PATCH 07/20] chore[vitess-cdc]: Rebase from latest master --- flink-sql-connector-vitess-cdc/pom.xml | 29 +++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index 25f4aed34e5..bd9ae202e74 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -29,7 +29,7 @@ under the License. - com.ververica + com.vinted flink-connector-vitess-cdc ${project.version} @@ -40,7 +40,7 @@ under the License. org.apache.maven.plugins maven-shade-plugin - 3.1.1 + 3.2.4 shade-flink @@ -52,7 +52,30 @@ under the License. false - *:* + + io.debezium:debezium-api + io.debezium:debezium-embedded + io.debezium:debezium-core + io.debezium:debezium-ddl-parser + io.debezium:debezium-connector-vitess + com.ververica:flink-connector-debezium + com.ververica:flink-connector-vitess-cdc + org.apache.kafka:* + com.google.guava:* + io.vitess:vitess-grpc-client + io.vitess:vitess-client + io.grpc:grpc-core + io.grpc:grpc-netty + io.grpc:grpc-stub + io.grpc:grpc-protobuf + io.grpc:grpc-protobuf-lite + io.grpc:grpc-context + io.grpc:grpc-api + com.google.protobuf:protobuf-java + com.google.code.gson:gson + io.opentracing.contrib:opentracing-grpc + io.perfmark:perfmark-api + com.github.luben:zstd-jni From cdf6f497a8181ea53c92261f2c5752c1599a7619 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Fri, 18 Mar 2022 11:48:10 +0200 Subject: [PATCH 08/20] chore[vitess-cdc]: Rebase from latest master --- docs/content/about.md | 2 ++ flink-sql-connector-vitess-cdc/pom.xml | 2 +- pom.xml | 12 ++++++++++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/content/about.md b/docs/content/about.md index 00400a2b1e6..a47badc19af 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -9,6 +9,7 @@ fully leverage the ability of Debezium. See more about what is [Debezium](https: ## Supported Connectors + | Connector | Database | Driver | |----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| | [mongodb-cdc](connectors/mongodb-cdc.md) |
  • [MongoDB](https://www.mongodb.com): 3.6, 4.x, 5.0 | MongoDB Driver: 4.3.1 | @@ -21,6 +22,7 @@ fully leverage the ability of Debezium. See more about what is [Debezium](https: | [db2-cdc](connectors/db2-cdc.md) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | DB2 Driver: 11.5.0.0 | | [Vitess-cdc](connectors/vitess-cdc.md) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.16 | +## Supported Flink Versions The following table shows the version mapping between Flink® CDC Connectors and Flink®: | Flink® CDC Version | Flink® Version | diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index bd9ae202e74..a14673d664c 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -29,7 +29,7 @@ under the License. - com.vinted + com.ververica flink-connector-vitess-cdc ${project.version} diff --git a/pom.xml b/pom.xml index 2936044a872..dc71b94d507 100644 --- a/pom.xml +++ b/pom.xml @@ -39,19 +39,19 @@ under the License. flink-connector-oracle-cdc flink-connector-mongodb-cdc flink-connector-oceanbase-cdc - flink-connector-vitess-cdc flink-connector-sqlserver-cdc flink-connector-tidb-cdc flink-connector-db2-cdc + flink-connector-vitess-cdc flink-sql-connector-mysql-cdc flink-sql-connector-postgres-cdc flink-sql-connector-mongodb-cdc flink-sql-connector-oracle-cdc flink-sql-connector-oceanbase-cdc - flink-sql-connector-vitess-cdc flink-sql-connector-sqlserver-cdc flink-sql-connector-tidb-cdc flink-sql-connector-db2-cdc + flink-sql-connector-vitess-cdc flink-cdc-e2e-tests @@ -89,6 +89,7 @@ under the License. ${java.version} ${java.version} 1.3 + 4.0.1 1.7.15 2.17.1 2.4.2 @@ -180,6 +181,13 @@ under the License. ${log4j.version} test + + + org.awaitility + awaitility + jar + ${version.awaitility} + From 445bf8f9ea77b84b1951d4c9693fb02668221ae0 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Wed, 15 Feb 2023 13:55:15 +0200 Subject: [PATCH 09/20] chore[vitess-cdc]: Rebase from latest master --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 33a0d21897d..808d7b6b017 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ This README is meant as a brief walkthrough on the core features of CDC Connecto | [sqlserver-cdc](docs/content/connectors/sqlserver-cdc.md) |
  • [Sqlserver](https://www.microsoft.com/sql-server): 2012, 2014, 2016, 2017, 2019 | JDBC Driver: 7.2.2.jre8 | | [tidb-cdc](docs/content/connectors/tidb-cdc.md) |
  • [TiDB](https://www.pingcap.com): 5.1.x, 5.2.x, 5.3.x, 5.4.x, 6.0.0 | JDBC Driver: 8.0.27 | | [Db2-cdc](docs/content/connectors/db2-cdc.md) |
  • [Db2](https://www.ibm.com/products/db2): 11.5 | Db2 Driver: 11.5.0.0 | +| [Vitess-cdc](connectors/vitess-cdc.md) |
  • [Vitess](https://vitess.io/): 8.0.x, 9.0.x | MySql JDBC Driver: 8.0.16 | ## Features From ac64e620ad82bb4c753358822f536d2a98453869 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Fri, 18 Mar 2022 16:29:43 +0200 Subject: [PATCH 10/20] fix: Address review comments --- .../cdc/connectors/vitess/VitessSource.java | 13 +++-- .../vitess/table/VitessTableFactory.java | 22 ++++----- .../vitess/table/VitessTableSource.java | 47 ++++++++++--------- .../connectors/vitess/VitessSourceTest.java | 2 +- .../cdc/connectors/vitess/VitessTestBase.java | 2 +- .../vitess/table/VitessConnectorITCase.java | 12 ++--- .../vitess/table/VitessTableFactoryTest.java | 8 ++-- .../{vitess => postgres}/DummyDocs.java | 2 +- 8 files changed, 50 insertions(+), 58 deletions(-) rename flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/{vitess => postgres}/DummyDocs.java (93%) diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java index 49c04baba77..cf52e2ed882 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessSource.java @@ -43,7 +43,7 @@ public static Builder builder() { public static class Builder { private String pluginName = "decoderbufs"; - private String slotName = "flink"; + private String name = "flink"; private int port = 15991; // default 15991 port private String hostname; private String keyspace; @@ -117,12 +117,11 @@ public Builder password(String password) { } /** - * The name of the Vitess logical decoding slot that was created for streaming changes from - * a particular plug-in for a particular database/schema. The server uses this slot to - * stream events to the connector that you are configuring. Default is "flink". + * Unique name for the connector. Attempting to register again with the same name will fail. + * This property is required by all Kafka Connect connectors. Default is "flink". */ - public Builder slotName(String slotName) { - this.slotName = slotName; + public Builder name(String name) { + this.name = name; return this; } @@ -191,7 +190,7 @@ public DebeziumSourceFunction build() { Properties props = new Properties(); props.setProperty("connector.class", VitessConnector.class.getCanonicalName()); props.setProperty("plugin.name", pluginName); - props.setProperty("slot.name", slotName); + props.setProperty("name", name); // hard code server name, because we don't need to distinguish it, docs: // Logical name that identifies and provides a namespace for the particular Vitess // Vtgate server/cluster being monitored. The logical name should be unique across diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java index c4ad9f2e296..f6c5ec66458 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableFactory.java @@ -21,12 +21,11 @@ import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.factories.DynamicTableFactory; import org.apache.flink.table.factories.DynamicTableSourceFactory; import org.apache.flink.table.factories.FactoryUtil; -import org.apache.flink.table.utils.TableSchemaUtils; import com.ververica.cdc.connectors.vitess.config.TabletType; import com.ververica.cdc.connectors.vitess.config.VtctldConfig; @@ -117,14 +116,14 @@ public class VitessTableFactory implements DynamicTableSourceFactory { .withDescription( "The name of the Vitess logical decoding plug-in installed on the server."); - private static final ConfigOption SLOT_NAME = - ConfigOptions.key("slot.name") + private static final ConfigOption NAME = + ConfigOptions.key("name") .stringType() .defaultValue("flink") .withDescription( - "The name of the Vitess logical decoding slot that was created for streaming changes " - + "from a particular plug-in for a particular database/schema. The server uses this slot " - + "to stream events to the connector that you are configuring. Default is \"flink\"."); + "Unique name for the connector." + + " Attempting to register again with the same name will fail. " + + "This property is required by all Kafka Connect connectors. Default is flink."); @Override public DynamicTableSource createDynamicTableSource(DynamicTableFactory.Context context) { @@ -148,9 +147,8 @@ public DynamicTableSource createDynamicTableSource(DynamicTableFactory.Context c .build(); TabletType tabletType = TabletType.valueOf(config.get(TABLET_TYPE)); String pluginName = config.get(DECODING_PLUGIN_NAME); - String slotName = config.get(SLOT_NAME); - TableSchema physicalSchema = - TableSchemaUtils.getPhysicalSchema(context.getCatalogTable().getSchema()); + String name = config.get(NAME); + ResolvedSchema physicalSchema = context.getCatalogTable().getResolvedSchema(); return new VitessTableSource( physicalSchema, @@ -163,7 +161,7 @@ public DynamicTableSource createDynamicTableSource(DynamicTableFactory.Context c vtctldConfig, tabletType, pluginName, - slotName, + name, getDebeziumProperties(context.getCatalogTable().getOptions())); } @@ -191,7 +189,7 @@ public Set> optionalOptions() { options.add(PASSWORD); options.add(TABLET_TYPE); options.add(DECODING_PLUGIN_NAME); - options.add(SLOT_NAME); + options.add(NAME); return options; } } diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java index cc5e791a74a..3cdd673b3a4 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/table/VitessTableSource.java @@ -19,7 +19,7 @@ package com.ververica.cdc.connectors.vitess.table; import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.table.connector.ChangelogMode; import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.connector.source.ScanTableSource; @@ -47,21 +47,21 @@ */ public class VitessTableSource implements ScanTableSource { - private final TableSchema physicalSchema; - private String pluginName; - private String slotName; - private int port; - private String hostname; - private String keyspace; - private String username; - private String password; - private String tableName; - private VtctldConfig vtctldConfig; - private TabletType tabletType; - private Properties dbzProperties; + private final ResolvedSchema physicalSchema; + private final String pluginName; + private final String name; + private final int port; + private final String hostname; + private final String keyspace; + private final String username; + private final String password; + private final String tableName; + private final VtctldConfig vtctldConfig; + private final TabletType tabletType; + private final Properties dbzProperties; public VitessTableSource( - TableSchema physicalSchema, + ResolvedSchema physicalSchema, int port, String hostname, String keyspace, @@ -71,7 +71,7 @@ public VitessTableSource( VtctldConfig vtctldConfig, TabletType tabletType, String pluginName, - String slotName, + String name, Properties dbzProperties) { this.physicalSchema = physicalSchema; this.port = port; @@ -83,7 +83,7 @@ public VitessTableSource( this.vtctldConfig = checkNotNull(vtctldConfig); this.tabletType = checkNotNull(tabletType); this.pluginName = checkNotNull(pluginName); - this.slotName = slotName; + this.name = name; this.dbzProperties = dbzProperties; } @@ -102,7 +102,8 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { RowType physicalDataType = (RowType) physicalSchema.toPhysicalRowDataType().getLogicalType(); TypeInformation typeInfo = - scanContext.createTypeInformation(physicalSchema.toRowDataType()); + scanContext.createTypeInformation(physicalSchema.toPhysicalRowDataType()); + DebeziumDeserializationSchema deserializer = RowDataDebeziumDeserializeSchema.newBuilder() .setPhysicalRowType(physicalDataType) @@ -121,7 +122,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { .tabletType(tabletType) .decodingPluginName(pluginName) .vtctldConfig(vtctldConfig) - .slotName(slotName) + .name(name) .debeziumProperties(dbzProperties) .deserializer(deserializer) .build(); @@ -141,7 +142,7 @@ public DynamicTableSource copy() { vtctldConfig, tabletType, pluginName, - slotName, + name, dbzProperties); } @@ -157,7 +158,7 @@ public boolean equals(Object o) { return port == that.port && Objects.equals(physicalSchema, that.physicalSchema) && Objects.equals(pluginName, that.pluginName) - && Objects.equals(slotName, that.slotName) + && Objects.equals(name, that.name) && Objects.equals(hostname, that.hostname) && Objects.equals(keyspace, that.keyspace) && Objects.equals(username, that.username) @@ -173,7 +174,7 @@ public int hashCode() { return Objects.hash( physicalSchema, pluginName, - slotName, + name, port, hostname, keyspace, @@ -193,8 +194,8 @@ public String toString() { + ", pluginName='" + pluginName + '\'' - + ", slotName='" - + slotName + + ", name='" + + name + '\'' + ", port=" + port diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java index bfacd0220a3..ac8f68628b2 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java @@ -61,7 +61,7 @@ public class VitessSourceTest extends VitessTestBase { @Before public void before() { - initializeMysqlTable("inventory"); + initializeTable("inventory"); } @Test diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java index d3958e549c2..c6530515a76 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessTestBase.java @@ -76,7 +76,7 @@ public Connection getJdbcConnection() throws SQLException { * Executes a JDBC statement using the default jdbc config without autocommitting the * connection. */ - protected void initializeMysqlTable(String sqlFile) { + protected void initializeTable(String sqlFile) { final String ddlFile = String.format("ddl/%s.sql", sqlFile); final URL ddlTestFile = VitessTestBase.class.getClassLoader().getResource(ddlFile); assertNotNull("Cannot locate " + ddlFile, ddlTestFile); diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java index 0324117b6e3..d072fbdc2d2 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java @@ -62,7 +62,7 @@ public void before() { @Test public void testConsumingAllEvents() throws SQLException, ExecutionException, InterruptedException { - initializeMysqlTable("inventory"); + initializeTable("inventory"); String sourceDDL = String.format( "CREATE TABLE debezium_source (" @@ -152,9 +152,9 @@ public void testConsumingAllEvents() result.getJobClient().get().cancel().get(); } - @Test // TODO add DATE, TIMESTAMP, TIME type mapping + @Test public void testAllTypes() throws Throwable { - initializeMysqlTable("column_type_test"); + initializeTable("column_type_test"); String sourceDDL = String.format( "CREATE TABLE full_types (\n" @@ -174,12 +174,6 @@ public void testAllTypes() throws Throwable { + " decimal_c DECIMAL(8, 4),\n" + " numeric_c DECIMAL(6, 0),\n" + " boolean_c BOOLEAN,\n" - // + " date_c DATE,\n" - // + " time_c TIME(0),\n" - // + " datetime3_c TIMESTAMP(3),\n" - // + " datetime6_c TIMESTAMP(6),\n" - // + " timestamp_c TIMESTAMP(0),\n" - // + " file_uuid BYTES,\n" + " primary key (`id`) not enforced" + ") WITH (" + " 'connector' = 'vitess-cdc'," diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java index 330576f6b32..c751fee0207 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessTableFactoryTest.java @@ -30,7 +30,6 @@ import org.apache.flink.table.connector.source.DynamicTableSource; import org.apache.flink.table.factories.Factory; import org.apache.flink.table.factories.FactoryUtil; -import org.apache.flink.table.utils.TableSchemaUtils; import org.apache.flink.util.ExceptionUtils; import com.ververica.cdc.connectors.vitess.config.TabletType; @@ -64,6 +63,7 @@ public class VitessTableFactoryTest { new ArrayList<>(), UniqueConstraint.primaryKey("pk", Arrays.asList("bbb", "aaa"))); + private static final String MY_SCHEMA = "public"; private static final String MY_LOCALHOST = "localhost"; private static final String MY_USERNAME = "flinkuser"; private static final String MY_PASSWORD = "flinkpw"; @@ -79,7 +79,7 @@ public void testCommonProperties() { DynamicTableSource actualSource = createTableSource(properties); VitessTableSource expectedSource = new VitessTableSource( - TableSchemaUtils.getPhysicalSchema(fromResolvedSchema(SCHEMA)), + SCHEMA, 15991, MY_LOCALHOST, MY_KEYSPACE, @@ -101,7 +101,7 @@ public void testOptionalProperties() { options.put("vtctl.port", "5445"); options.put("decoding.plugin.name", "wal2json"); options.put("debezium.snapshot.mode", "never"); - options.put("slot.name", "flink"); + options.put("name", "flink"); options.put("tablet-type", "MASTER"); options.put("username", MY_USERNAME); options.put("password", MY_PASSWORD); @@ -111,7 +111,7 @@ public void testOptionalProperties() { dbzProperties.put("snapshot.mode", "never"); VitessTableSource expectedSource = new VitessTableSource( - TableSchemaUtils.getPhysicalSchema(fromResolvedSchema(SCHEMA)), + SCHEMA, 5444, MY_LOCALHOST, MY_KEYSPACE, diff --git a/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java b/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java similarity index 93% rename from flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java rename to flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java index 46c174dc6ed..deb9e330d14 100644 --- a/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/vitess/DummyDocs.java +++ b/flink-sql-connector-postgres-cdc/src/main/java/com/ververica/cdc/connectors/postgres/DummyDocs.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.ververica.cdc.connectors.vitess; +package com.ververica.cdc.connectors.postgres; /** This is used to generate a dummy docs jar for this module to pass OSS repository rule. */ public class DummyDocs {} From 92f0a92ce63bf6e8e7e22d80581138b3bd684596 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Tue, 24 May 2022 13:43:55 +0300 Subject: [PATCH 11/20] chore[vitess-cdc]: Rebase from latest master --- flink-connector-vitess-cdc/pom.xml | 2 +- flink-sql-connector-vitess-cdc/pom.xml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index 29a3408ff8a..7d8ed63ebd8 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.2-SNAPSHOT + 2.3-SNAPSHOT 4.0.0 diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index a14673d664c..aa08406eded 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.2-SNAPSHOT + 2.3-SNAPSHOT 4.0.0 @@ -52,7 +52,6 @@ under the License. false - io.debezium:debezium-api io.debezium:debezium-embedded io.debezium:debezium-core From 9676b994ff88560f835281b13f226ba27e05dce1 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Wed, 29 Jun 2022 16:40:24 +0300 Subject: [PATCH 12/20] chore: Remove explicit log4 config --- flink-connector-vitess-cdc/pom.xml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index 7d8ed63ebd8..b504c152313 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -161,30 +161,6 @@ under the License. test - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - test - - - - - org.apache.maven.plugins - maven-surefire-plugin - - - 1 - -Xms256m -Xmx2048m -Dlog4j.configurationFile=${log4j.configuration} - -Dmvn.forkNumber=${surefire.forkNumber} -XX:-UseGCOverheadLimit - - - - - - From 2aeb232f1d913115d237fffefbbdc9df69e27bfd Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Fri, 8 Jul 2022 11:21:33 +0300 Subject: [PATCH 13/20] chore[vitess-cdc]: Rebase from latest master --- flink-connector-vitess-cdc/pom.xml | 6 +++--- .../ververica/cdc/connectors/vitess/VitessValidator.java | 2 +- .../vervetica/cdc/connectors/vitess/VitessSourceTest.java | 6 ++++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index b504c152313..cde12b30820 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -79,14 +79,14 @@ under the License. org.apache.flink - flink-table-planner-blink_${scala.binary.version} + flink-table-planner_${scala.binary.version} ${flink.version} test org.apache.flink - flink-table-runtime-blink_${scala.binary.version} + flink-table-runtime_${scala.binary.version} ${flink.version} test @@ -132,7 +132,7 @@ under the License. org.apache.flink - flink-table-planner-blink_${scala.binary.version} + flink-table-planner_${scala.binary.version} ${flink.version} test-jar test diff --git a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java index 95cd61356dd..99a429c3063 100644 --- a/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java +++ b/flink-connector-vitess-cdc/src/main/java/com/ververica/cdc/connectors/vitess/VitessValidator.java @@ -18,7 +18,7 @@ package com.ververica.cdc.connectors.vitess; -import org.apache.flink.shaded.guava18.com.google.common.collect.Maps; +import org.apache.flink.shaded.guava30.com.google.common.collect.Maps; import com.ververica.cdc.debezium.Validator; import io.debezium.connector.vitess.VitessConnector; diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java index ac8f68628b2..4dfe11f944d 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java @@ -47,6 +47,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.OptionalLong; import java.util.Properties; import java.util.Set; import java.util.concurrent.LinkedBlockingQueue; @@ -297,6 +298,11 @@ public boolean isRestored() { return isRestored; } + @Override + public OptionalLong getRestoredCheckpointId() { + throw new UnsupportedOperationException(); + } + @Override public OperatorStateStore getOperatorStateStore() { return operatorStateStore; From cf94708b983b6287ad294f6a2e37b1c366d972e4 Mon Sep 17 00:00:00 2001 From: gintarasm Date: Fri, 22 Jul 2022 16:35:44 +0300 Subject: [PATCH 14/20] chore[vitess-cdc]: relocate grpc and netty --- flink-sql-connector-vitess-cdc/pom.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index aa08406eded..71a8fc1d5ed 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -77,6 +77,16 @@ under the License. com.github.luben:zstd-jni + + + io.grpc + vitess.shaded.io.grpc + + + io.netty + vitess.shaded.io.netty + + org.apache.kafka:* From ba7f32a3799cd4cee5b29afabacc28ecacbbb173 Mon Sep 17 00:00:00 2001 From: gintarasm Date: Fri, 22 Jul 2022 16:35:44 +0300 Subject: [PATCH 15/20] chore[vitess-cdc]: relocate grpc and netty --- flink-sql-connector-vitess-cdc/pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index 71a8fc1d5ed..f1661ea5df3 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -80,11 +80,16 @@ under the License. io.grpc - vitess.shaded.io.grpc + com.ververica.cdc.connectors.shaded.io.grpc io.netty vitess.shaded.io.netty + com.ververica.cdc.connectors.shaded.io.grpc + + + io.netty + com.ververica.cdc.connectors.shaded.io.netty From 230b8c6208f7909eb214fbcf4fe7851ca96d1807 Mon Sep 17 00:00:00 2001 From: gintarasm Date: Mon, 25 Jul 2022 11:10:17 +0300 Subject: [PATCH 16/20] chore[vitess-cdc]: relocate google package --- flink-sql-connector-vitess-cdc/pom.xml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index f1661ea5df3..787590bf7f9 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -84,12 +84,11 @@ under the License. io.netty - vitess.shaded.io.netty - com.ververica.cdc.connectors.shaded.io.grpc + com.ververica.cdc.connectors.shaded.io.netty - io.netty - com.ververica.cdc.connectors.shaded.io.netty + com.google + com.ververica.cdc.connectors.shaded.com.google From 6e0ee7508a0be070046eb46150b6af2f2e63cdfa Mon Sep 17 00:00:00 2001 From: gintarasm Date: Wed, 27 Jul 2022 16:26:27 +0300 Subject: [PATCH 17/20] chore[vitess-cdc]: promote transitive dependencies --- flink-sql-connector-vitess-cdc/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index 787590bf7f9..91c25086b5b 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -50,6 +50,7 @@ under the License. false + true io.debezium:debezium-api From e4c441555aa23f0f07c78dfb9484b9fbcddd9b61 Mon Sep 17 00:00:00 2001 From: gintarasm Date: Thu, 28 Jul 2022 14:55:38 +0300 Subject: [PATCH 18/20] chore[vitess-cdc]: fix io grpc relocation --- flink-sql-connector-vitess-cdc/pom.xml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index 91c25086b5b..8eac6f239f9 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -50,7 +50,9 @@ under the License. false - true + + + io.debezium:debezium-api @@ -64,6 +66,14 @@ under the License. com.google.guava:* io.vitess:vitess-grpc-client io.vitess:vitess-client + io.netty:netty-handler + io.netty:netty-resolver + io.netty:netty-transport + io.netty:netty-buffer + io.netty:netty-codec-http2 + io.netty:netty-codec-http + io.netty:netty-codec + io.netty:netty-common io.grpc:grpc-core io.grpc:grpc-netty io.grpc:grpc-stub From a2330e55badb8aa5014856ebe744a58f386e8c16 Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Thu, 3 Nov 2022 12:20:51 +0200 Subject: [PATCH 19/20] feat[vitess]: Upgrade to flink 1.15 --- flink-connector-vitess-cdc/pom.xml | 8 ++++---- .../connectors/vitess/table/VitessConnectorITCase.java | 3 +-- flink-sql-connector-vitess-cdc/pom.xml | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flink-connector-vitess-cdc/pom.xml b/flink-connector-vitess-cdc/pom.xml index cde12b30820..446c9dd5d21 100644 --- a/flink-connector-vitess-cdc/pom.xml +++ b/flink-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.3-SNAPSHOT + 2.4-SNAPSHOT 4.0.0 @@ -86,14 +86,14 @@ under the License. org.apache.flink - flink-table-runtime_${scala.binary.version} + flink-table-runtime ${flink.version} test org.apache.flink - flink-test-utils_${scala.binary.version} + flink-test-utils ${flink.version} test @@ -108,7 +108,7 @@ under the License. org.apache.flink - flink-streaming-java_${scala.binary.version} + flink-streaming-java ${flink.version} test-jar test diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java index d072fbdc2d2..8c59b4c39c7 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/table/VitessConnectorITCase.java @@ -50,8 +50,7 @@ public class VitessConnectorITCase extends VitessTestBase { StreamExecutionEnvironment.getExecutionEnvironment(); private final StreamTableEnvironment tEnv = StreamTableEnvironment.create( - env, - EnvironmentSettings.newInstance().useBlinkPlanner().inStreamingMode().build()); + env, EnvironmentSettings.newInstance().inStreamingMode().build()); @Before public void before() { diff --git a/flink-sql-connector-vitess-cdc/pom.xml b/flink-sql-connector-vitess-cdc/pom.xml index 8eac6f239f9..d1e6a129f60 100644 --- a/flink-sql-connector-vitess-cdc/pom.xml +++ b/flink-sql-connector-vitess-cdc/pom.xml @@ -21,7 +21,7 @@ under the License. flink-cdc-connectors com.ververica - 2.3-SNAPSHOT + 2.4-SNAPSHOT 4.0.0 From 9313a1d106c70a37a1d821ccd02c80313f79afbf Mon Sep 17 00:00:00 2001 From: Simonas Gelazevicius Date: Wed, 15 Mar 2023 14:12:00 +0200 Subject: [PATCH 20/20] fix[vitess]: Fix failing vitess tests --- .../com/vervetica/cdc/connectors/vitess/VitessSourceTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java index 4dfe11f944d..fcd878403f0 100644 --- a/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java +++ b/flink-connector-vitess-cdc/src/test/java/com/vervetica/cdc/connectors/vitess/VitessSourceTest.java @@ -122,6 +122,9 @@ public void go() throws Exception { // ... statement.execute( "ALTER TABLE test.products ADD COLUMN volume FLOAT, ADD COLUMN alias VARCHAR(30) NULL"); + + // Vitess schema change has eventual consistency, wait few seconds. + Thread.sleep(5000); statement.execute("UPDATE test.products SET volume=13.5 WHERE id=2001"); records = drain(sourceContext, 1); assertUpdate(records.get(0), "id", 2001);