diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf index c3fde6995cd..edd78432862 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-assert-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -44,10 +44,10 @@ source { transform { Filter { - source_table_name = "fake" - result_table_name = "fake1" - fields = ["name", "age"] - } + source_table_name = "fake" + result_table_name = "fake1" + fields = ["name", "age"] + } } sink { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf index 4f42eb8625f..f7e800e99b4 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cassandra-e2e/src/test/resources/cassandra_to_cassandra.conf @@ -23,15 +23,15 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - Cassandra { - host = "cassandra:9042" - username = "" - password = "" - datacenter = "datacenter1" - keyspace = "test" - cql = "select * from source_table" - result_table_name = "source_table" - } + Cassandra { + host = "cassandra:9042" + username = "" + password = "" + datacenter = "datacenter1" + keyspace = "test" + cql = "select * from source_table" + result_table_name = "source_table" + } } transform { @@ -48,5 +48,5 @@ sink { keyspace = "test" async_write = "true" table = "sink_table" - } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql.conf index cf0c2c908d8..7b92bd986ac 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_to_mysql.conf @@ -41,16 +41,16 @@ transform { sink { jdbc { - source_table_name = "customers_mysql_cdc" - url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" - driver = "com.mysql.cj.jdbc.Driver" - user = "st_user" - password = "seatunnel" + source_table_name = "customers_mysql_cdc" + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + driver = "com.mysql.cj.jdbc.Driver" + user = "st_user" + password = "seatunnel" - generate_sink_sql = true - # You need to configure both database and table - database = mysql_cdc - table = mysql_cdc_e2e_sink_table - primary_keys = ["id"] + generate_sink_sql = true + # You need to configure both database and table + database = mysql_cdc + table = mysql_cdc_e2e_sink_table + primary_keys = ["id"] } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_console.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_console.conf index 6a37236319a..c4ac06877b1 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_console.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-sqlserver-e2e/src/test/resources/sqlservercdc_to_console.conf @@ -42,18 +42,18 @@ transform { sink { console { - source_table_name = "customers" + source_table_name = "customers" } Jdbc { - source_table_name = "customers" - driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" - url = "jdbc:sqlserver://sqlserver-host:1433;encrypt=false" - user = "sa" - password = "Password!" - generate_sink_sql = true - database = "" - table = "column_type_test.dbo.full_types_sink" - batch_size = 1 - primary_keys = ["id"] - } + source_table_name = "customers" + driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver" + url = "jdbc:sqlserver://sqlserver-host:1433;encrypt=false" + user = "sa" + password = "Password!" + generate_sink_sql = true + database = "" + table = "column_type_test.dbo.full_types_sink" + batch_size = 1 + primary_keys = ["id"] + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case1.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case1.conf index c5060bba7e1..e32b11089bb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case1.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case1.conf @@ -24,63 +24,63 @@ env { } source { - FakeSource { - schema = { - fields { - pk_id = bigint - name = string - score = int - } - } - rows = [ - { - kind = INSERT - fields = [1, "A", 100] - }, - { - kind = INSERT - fields = [2, "B", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - } - { - kind = UPDATE_BEFORE - fields = [1, "A", 100] - }, - { - kind = UPDATE_AFTER - fields = [1, "A_1", 100] - }, - { - kind = DELETE - fields = [2, "B", 100] - } - ] + FakeSource { + schema = { + fields { + pk_id = bigint + name = string + score = int + } } + rows = [ + { + kind = INSERT + fields = [1, "A", 100] + }, + { + kind = INSERT + fields = [2, "B", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + } + { + kind = UPDATE_BEFORE + fields = [1, "A", 100] + }, + { + kind = UPDATE_AFTER + fields = [1, "A_1", 100] + }, + { + kind = DELETE + fields = [2, "B", 100] + } + ] + } } sink { - Clickhouse { - host = "clickhouse:8123" - database = "default" - table = "sink_table" - username = "default" - password = "" + Clickhouse { + host = "clickhouse:8123" + database = "default" + table = "sink_table" + username = "default" + password = "" - primary_key = "pk_id" - } + primary_key = "pk_id" + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case2.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case2.conf index f1a630a4c5a..dead81502fe 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case2.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_sink_cdc_changelog_case2.conf @@ -24,64 +24,64 @@ env { } source { - FakeSource { - schema = { - fields { - pk_id = bigint - name = string - score = int - } - } - rows = [ - { - kind = INSERT - fields = [1, "A", 100] - }, - { - kind = INSERT - fields = [2, "B", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - } - { - kind = UPDATE_BEFORE - fields = [1, "A", 100] - }, - { - kind = UPDATE_AFTER - fields = [1, "A_1", 100] - }, - { - kind = DELETE - fields = [2, "B", 100] - } - ] + FakeSource { + schema = { + fields { + pk_id = bigint + name = string + score = int + } } + rows = [ + { + kind = INSERT + fields = [1, "A", 100] + }, + { + kind = INSERT + fields = [2, "B", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + } + { + kind = UPDATE_BEFORE + fields = [1, "A", 100] + }, + { + kind = UPDATE_AFTER + fields = [1, "A_1", 100] + }, + { + kind = DELETE + fields = [2, "B", 100] + } + ] + } } sink { - Clickhouse { - host = "clickhouse:8123" - database = "default" - table = "sink_table" - username = "default" - password = "" + Clickhouse { + host = "clickhouse:8123" + database = "default" + table = "sink_table" + username = "default" + password = "" - primary_key = "pk_id" - allow_experimental_lightweight_delete = true - } + primary_key = "pk_id" + allow_experimental_lightweight_delete = true + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf index 1137c7c9375..d9b5c86e8f3 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-clickhouse-e2e/src/test/resources/clickhouse_to_clickhouse.conf @@ -26,14 +26,14 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - Clickhouse { - host = "clickhouse:8123" - database = "default" - sql = "select * from source_table" - username = "default" - password = "" - result_table_name = "source_table" - } + Clickhouse { + host = "clickhouse:8123" + database = "default" + sql = "select * from source_table" + username = "default" + password = "" + result_table_name = "source_table" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/connector-v2/source/ClickhouseSource } @@ -45,7 +45,7 @@ sink { table = "sink_table" username = "default" password = "" - } + } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, # please go to https://seatunnel.apache.org/docs/connector-v2/sink diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf index 7defe940551..2bd653dfdfc 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-datahub-e2e/src/test/resources/fakesource_to_datahub.conf @@ -37,12 +37,12 @@ transform { sink { DataHub { - endpoint="xxx" - accessId="xxx" - accessKey="xxx" - project="xxx" - topic="xxx" - timeout=3000 - retryTimes=3 - } + endpoint = "xxx" + accessId = "xxx" + accessKey = "xxx" + project = "xxx" + topic = "xxx" + timeout = 3000 + retryTimes = 3 + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/resources/write-cdc-changelog-to-doris.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/resources/write-cdc-changelog-to-doris.conf index c6e423ae102..dde7dca627b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/resources/write-cdc-changelog-to-doris.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-doris-e2e/src/test/resources/write-cdc-changelog-to-doris.conf @@ -16,60 +16,60 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { - FakeSource { - schema = { - fields { - pk_id = bigint - name = string - score = int - } - } - rows = [ - { - kind = INSERT - fields = [1, "A", 100] - }, - { - kind = INSERT - fields = [2, "B", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = UPDATE_BEFORE - fields = [1, "A", 100] - }, - { - kind = UPDATE_AFTER - fields = [1, "A_1", 100] - }, - { - kind = DELETE - fields = [2, "B", 100] - } - ] + FakeSource { + schema = { + fields { + pk_id = bigint + name = string + score = int + } } + rows = [ + { + kind = INSERT + fields = [1, "A", 100] + }, + { + kind = INSERT + fields = [2, "B", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = UPDATE_BEFORE + fields = [1, "A", 100] + }, + { + kind = UPDATE_AFTER + fields = [1, "A_1", 100] + }, + { + kind = DELETE + fields = [2, "B", 100] + } + ] + } } sink { - Doris { - fenodes = "doris_cdc_e2e:8030" - username = root - password = "" - table.identifier = "test.e2e_table_sink" - sink.label-prefix = "test-cdc" - sink.enable-2pc = "false" - sink.enable-delete = "true" - doris.config { - format="json" - read_json_by_line="true" - } + Doris { + fenodes = "doris_cdc_e2e:8030" + username = root + password = "" + table.identifier = "test.e2e_table_sink" + sink.label-prefix = "test-cdc" + sink.enable-2pc = "false" + sink.enable-delete = "true" + doris.config { + format = "json" + read_json_by_line = "true" } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/resources/elasticsearch/elasticsearch_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/resources/elasticsearch/elasticsearch_source_and_sink.conf index cfca9607359..1702f59893f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/resources/elasticsearch/elasticsearch_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-elasticsearch-e2e/src/test/resources/elasticsearch/elasticsearch_source_and_sink.conf @@ -28,48 +28,48 @@ env { } source { - Elasticsearch { - hosts = ["https://elasticsearch:9200"] - username = "elastic" - password = "elasticsearch" - tls_verify_certificate = false - tls_verify_hostname = false + Elasticsearch { + hosts = ["https://elasticsearch:9200"] + username = "elastic" + password = "elasticsearch" + tls_verify_certificate = false + tls_verify_hostname = false - index = "st_index" - query = {"range":{"c_int":{"gte":10,"lte":20}}} - schema = { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(2, 1)" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } - } + index = "st_index" + query = {"range": {"c_int": {"gte": 10, "lte": 20}}} + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + } } + } } transform { } sink { - Elasticsearch{ - hosts = ["https://elasticsearch:9200"] - username = "elastic" - password = "elasticsearch" - tls_verify_certificate = false - tls_verify_hostname = false + Elasticsearch { + hosts = ["https://elasticsearch:9200"] + username = "elastic" + password = "elasticsearch" + tls_verify_certificate = false + tls_verify_hostname = false - index = "st_index2" - index_type = "st" - } + index = "st_index2" + index_type = "st" + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hbase-e2e/src/test/resources/fake-to-hbase.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hbase-e2e/src/test/resources/fake-to-hbase.conf index 04e2507560c..c34f13be837 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hbase-e2e/src/test/resources/fake-to-hbase.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hbase-e2e/src/test/resources/fake-to-hbase.conf @@ -16,35 +16,35 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { - FakeSource { - row.num = 5 - schema { - fields { - name = string - age = int - c_tinyint = tinyint - c_smallint = smallint - c_bigint = bigint - c_float = float - c_double = double - c_boolean = boolean - } - } + FakeSource { + row.num = 5 + schema { + fields { + name = string + age = int + c_tinyint = tinyint + c_smallint = smallint + c_bigint = bigint + c_float = float + c_double = double + c_boolean = boolean + } } + } } sink { - Hbase { - zookeeper_quorum = "hbase-e2e:2181" - table = "seatunnel_test" - rowkey_column = ["name"] - family_name { - all_columns = info - } + Hbase { + zookeeper_quorum = "hbase-e2e:2181" + table = "seatunnel_test" + rowkey_column = ["name"] + family_name { + all_columns = info } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_multilinejson_to_assert.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_multilinejson_to_assert.conf index c7c292c6b27..a0ad32a8ae6 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_multilinejson_to_assert.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-http-e2e/src/test/resources/http_multilinejson_to_assert.conf @@ -21,20 +21,20 @@ env { } source { - Http { - result_table_name = "http" - url = "http://mockserver:1080/example/httpMultiLine" - method = "GET" - format = "json" - schema = { - fields { - name = string - age = int - salary = int - } - } - enable_multi_lines=true - } + Http { + result_table_name = "http" + url = "http://mockserver:1080/example/httpMultiLine" + method = "GET" + format = "json" + schema = { + fields { + name = string + age = int + salary = int + } + } + enable_multi_lines = true + } } sink { @@ -42,14 +42,14 @@ sink { source_table_name = "http" } Assert { - source_table_name = "http" - rules { - row_rules = [ - { - rule_type = MAX_ROW - rule_value = 2 - } - ] - } - } + source_table_name = "http" + rules { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 2 + } + ] + } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf index 400d4c64080..0eadab98f8d 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-e2e/src/test/resources/iceberg/iceberg_source.conf @@ -26,23 +26,23 @@ env { source { Iceberg { schema { - fields { - f2 = "boolean" - f1 = "bigint" - f3 = "int" - f4 = "bigint" - f5 = "float" - f6 = "double" - f7 = "date" - f9 = "timestamp" - f10 = "timestamp" - f11 = "string" - f12 = "bytes" - f13 = "bytes" - f14 = "decimal(19,9)" - f15 = "array" - f16 = "map" - } + fields { + f2 = "boolean" + f1 = "bigint" + f3 = "int" + f4 = "bigint" + f5 = "float" + f6 = "double" + f7 = "date" + f9 = "timestamp" + f10 = "timestamp" + f11 = "string" + f12 = "bytes" + f13 = "bytes" + f14 = "decimal(19,9)" + f15 = "array" + f16 = "map" + } } catalog_name = "seatunnel" catalog_type = "hadoop" diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf index d0592de23e5..5b3bb18364f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iceberg-hadoop3-e2e/src/test/resources/iceberg/iceberg_source.conf @@ -19,73 +19,73 @@ ###### env { - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local - job.mode = "BATCH" + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + job.mode = "BATCH" } source { - Iceberg { - schema { - fields { - f2 = "boolean" - f1 = "bigint" - f3 = "int" - f4 = "bigint" - f5 = "float" - f6 = "double" - f7 = "date" - f9 = "timestamp" - f10 = "timestamp" - f11 = "string" - f12 = "bytes" - f13 = "bytes" - f14 = "decimal(19,9)" - f15 = "array" - f16 = "map" - } - } - catalog_name = "seatunnel" - catalog_type = "hadoop" - warehouse = "file:///tmp/seatunnel/iceberg/hadoop3/" - namespace = "database1" - table = "source" - result_table_name = "iceberg" + Iceberg { + schema { + fields { + f2 = "boolean" + f1 = "bigint" + f3 = "int" + f4 = "bigint" + f5 = "float" + f6 = "double" + f7 = "date" + f9 = "timestamp" + f10 = "timestamp" + f11 = "string" + f12 = "bytes" + f13 = "bytes" + f14 = "decimal(19,9)" + f15 = "array" + f16 = "map" + } } + catalog_name = "seatunnel" + catalog_type = "hadoop" + warehouse = "file:///tmp/seatunnel/iceberg/hadoop3/" + namespace = "database1" + table = "source" + result_table_name = "iceberg" + } } transform { } sink { - Console { - source_table_name = "iceberg" - } - Assert { - source_table_name = "iceberg" - rules = { - field_rules = [ - { - field_name = f1 - field_type = long - field_value = [ - { - rule_type = NOT_NULL - }, - { - rule_type = MIN - rule_value = 0 - }, - { - rule_type = MAX - rule_value = 99 - } - ] - } - ] + Console { + source_table_name = "iceberg" + } + Assert { + source_table_name = "iceberg" + rules = { + field_rules = [ + { + field_name = f1 + field_type = long + field_value = [ + { + rule_type = NOT_NULL + }, + { + rule_type = MIN + rule_value = 0 + }, + { + rule_type = MAX + rule_value = 99 + } + ] } - } + ] + } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/resources/influxdb-to-influxdb.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/resources/influxdb-to-influxdb.conf index 59dc743848a..be8762c948b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/resources/influxdb-to-influxdb.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-influxdb-e2e/src/test/resources/influxdb-to-influxdb.conf @@ -16,45 +16,45 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { - InfluxDB { - url = "http://influxdb-host:8086" - sql = "select label, c_string, c_double, c_bigint, c_float, c_int, c_smallint, c_boolean from source" - database = "test" - upper_bound = 99 - lower_bound = 0 - partition_num = 4 - split_column = "c_int" - schema { - fields { - label = STRING - c_string = STRING - c_double = DOUBLE - c_bigint = BIGINT - c_float = FLOAT - c_int = INT - c_smallint = SMALLINT - c_boolean = BOOLEAN - time = BIGINT - } - } + InfluxDB { + url = "http://influxdb-host:8086" + sql = "select label, c_string, c_double, c_bigint, c_float, c_int, c_smallint, c_boolean from source" + database = "test" + upper_bound = 99 + lower_bound = 0 + partition_num = 4 + split_column = "c_int" + schema { + fields { + label = STRING + c_string = STRING + c_double = DOUBLE + c_bigint = BIGINT + c_float = FLOAT + c_int = INT + c_smallint = SMALLINT + c_boolean = BOOLEAN + time = BIGINT + } } + } } transform { } sink { - InfluxDB { - url = "http://influxdb-host:8086" - database = "test" - measurement = "sink" - key_time = "time" - key_tags = ["label"] - batch_size = 1 - } + InfluxDB { + url = "http://influxdb-host:8086" + database = "test" + measurement = "sink" + key_time = "time" + key_tags = ["label"] + batch_size = 1 + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/resources/iotdb/iotdb_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/resources/iotdb/iotdb_source_to_sink.conf index 10e4eef929c..dd10ac9ac72 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/resources/iotdb/iotdb_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-iotdb-e2e/src/test/resources/iotdb/iotdb_source_to_sink.conf @@ -19,63 +19,63 @@ ###### env { - # You can set flink configuration here - execution.parallelism = 2 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" + # You can set flink configuration here + execution.parallelism = 2 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { - IoTDB { - result_table_name = "fake" + IoTDB { + result_table_name = "fake" - node_urls = "flink_e2e_iotdb_sink:6667" - username = "root" - password = "root" - sql = "SELECT c_string, c_boolean, c_tinyint, c_smallint, c_int, c_bigint, c_float, c_double FROM root.source_group.* WHERE time < 4102329600000 align by device" - lower_bound = 1 - upper_bound = 4102329600000 - num_partitions = 10 - schema { - fields { - ts = timestamp - device_name = string - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - } - } + node_urls = "flink_e2e_iotdb_sink:6667" + username = "root" + password = "root" + sql = "SELECT c_string, c_boolean, c_tinyint, c_smallint, c_int, c_bigint, c_float, c_double FROM root.source_group.* WHERE time < 4102329600000 align by device" + lower_bound = 1 + upper_bound = 4102329600000 + num_partitions = 10 + schema { + fields { + ts = timestamp + device_name = string + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + } } + } } transform { - Replace { - source_table_name = "fake" - result_table_name = "fake1" - replace_field = "device_name" - pattern = "root.source_group" - replacement = "root.sink_group" - is_regex = false - replace_first = true - } + Replace { + source_table_name = "fake" + result_table_name = "fake1" + replace_field = "device_name" + pattern = "root.source_group" + replacement = "root.sink_group" + is_regex = false + replace_first = true + } } sink { - IoTDB { - source_table_name = "fake1" - node_urls = ["flink_e2e_iotdb_sink:6667"] - username = "root" - password = "root" - key_device = "device_name" - key_timestamp = "ts" - key_measurement_fields = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - batch_size = 1 - batch_interval_ms = 10 - } + IoTDB { + source_table_name = "fake1" + node_urls = ["flink_e2e_iotdb_sink:6667"] + username = "root" + password = "root" + key_device = "device_name" + key_timestamp = "ts" + key_measurement_fields = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + batch_size = 1 + batch_interval_ms = 10 + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink.conf index eb4d1ac4b9f..00871f0f095 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink.conf @@ -16,38 +16,38 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { - jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false" - driver = "com.mysql.cj.jdbc.Driver" - connection_check_timeout_sec = 100 - user = "root" - password = "Abc!@#135_seatunnel" + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false" + driver = "com.mysql.cj.jdbc.Driver" + connection_check_timeout_sec = 100 + user = "root" + password = "Abc!@#135_seatunnel" - query = "select * from source;" - } + query = "select * from source;" + } } transform { } sink { - jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" - query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, + query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, c_datetime, c_timestamp, c_tinyblob, c_mediumblob, c_blob, c_longblob, c_varbinary, c_binary, c_year, c_int_unsigned, c_integer_unsigned) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""" - } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel.conf index 7594db6ba8a..bd1ad4f5469 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel.conf @@ -20,37 +20,37 @@ env { job.mode = "BATCH" } -source{ - jdbc{ - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" - query = "select * from source" - partition_column = "c_integer" - partition_num = 3 +source { + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "select * from source" + partition_column = "c_integer" + partition_num = 3 - result_table_name = "jdbc" - } + result_table_name = "jdbc" + } } transform { } sink { - jdbc { + jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" - connection_check_timeout_sec = 100 - query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + connection_check_timeout_sec = 100 + query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, c_datetime, c_timestamp, c_tinyblob, c_mediumblob, c_blob, c_longblob, c_varbinary, c_binary, c_year, c_int_unsigned, c_integer_unsigned) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""" - } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel_upper_lower.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel_upper_lower.conf index dedfd0eefb1..a9e54d7acdb 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel_upper_lower.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_parallel_upper_lower.conf @@ -20,38 +20,38 @@ env { job.mode = "BATCH" } -source{ - jdbc{ - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" - query = "select * from source" - partition_column = "c_int" - result_table_name = "jdbc" - partition_lower_bound = 1 - partition_upper_bound = 50 - partition_num = 5 - } +source { + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "select * from source" + partition_column = "c_int" + result_table_name = "jdbc" + partition_lower_bound = 1 + partition_upper_bound = 50 + partition_num = 5 + } } transform { } sink { - jdbc { + jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" - connection_check_timeout_sec = 100 - query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + connection_check_timeout_sec = 100 + query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, c_datetime, c_timestamp, c_tinyblob, c_mediumblob, c_blob, c_longblob, c_varbinary, c_binary, c_year, c_int_unsigned, c_integer_unsigned) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""" - } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_xa.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_xa.conf index e44bf1c7d73..bf0d3afab15 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_xa.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-1/src/test/resources/jdbc_mysql_source_and_sink_xa.conf @@ -22,27 +22,27 @@ env { } source { - jdbc{ - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" - query = "select * from source" -} + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" + query = "select * from source" + } } transform { } sink { - jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" - driver = "com.mysql.cj.jdbc.Driver" - user = "root" - password = "Abc!@#135_seatunnel" + jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + user = "root" + password = "Abc!@#135_seatunnel" - max_retries = 0 - query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, + max_retries = 0 + query = """insert into sink (c_bit_1, c_bit_8, c_bit_16, c_bit_32, c_bit_64, c_boolean, c_tinyint, c_tinyint_unsigned, c_smallint, c_smallint_unsigned, c_mediumint, c_mediumint_unsigned, c_int, c_integer, c_bigint, c_bigint_unsigned, c_decimal, c_decimal_unsigned, c_float, c_float_unsigned, c_double, c_double_unsigned, c_char, c_tinytext, c_mediumtext, c_text, c_varchar, c_json, c_longtext, c_date, @@ -50,11 +50,11 @@ sink { c_binary, c_year, c_int_unsigned, c_integer_unsigned) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);""" - # Non-root users need to grant XA_RECOVER_ADMIN permission on is_exactly_once = "true" - is_exactly_once = "true" + # Non-root users need to grant XA_RECOVER_ADMIN permission on is_exactly_once = "true" + is_exactly_once = "true" - xa_data_source_class_name = "com.mysql.cj.jdbc.MysqlXADataSource" - max_commit_attempts = 3 - transaction_timeout_sec = 86400 - } + xa_data_source_class_name = "com.mysql.cj.jdbc.MysqlXADataSource" + max_commit_attempts = 3 + transaction_timeout_sec = 86400 + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/doris-jdbc-to-doris.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/doris-jdbc-to-doris.conf index ce07bbe263b..6cb63b0ac8c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/doris-jdbc-to-doris.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/doris-jdbc-to-doris.conf @@ -16,8 +16,8 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { @@ -34,16 +34,16 @@ transform { } sink { - Doris { - fenodes = "doris_e2e:8030" - username = root - password = "" - table.identifier = "test.e2e_table_sink" - sink.enable-2pc = "false" - sink.label-prefix = "test_doris" - doris.config = { - format="json" - read_json_by_line="true" - } + Doris { + fenodes = "doris_e2e:8030" + username = root + password = "" + table.identifier = "test.e2e_table_sink" + sink.enable-2pc = "false" + sink.label-prefix = "test_doris" + doris.config = { + format = "json" + read_json_by_line = "true" } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_doris_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_doris_source_and_sink.conf index 7a2da5fbad0..32e2c7b9824 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_doris_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_doris_source_and_sink.conf @@ -16,8 +16,8 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_greenplum_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_greenplum_source_and_sink.conf index c8960b28adf..259e87aad1b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_greenplum_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_greenplum_source_and_sink.conf @@ -19,42 +19,42 @@ ###### env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - Jdbc { - driver = org.postgresql.Driver - url = "jdbc:postgresql://flink_e2e_greenplum:5432/testdb" - user = tester - password = pivotal - query = "select age, name from source" - } + # This is a example source plugin **only for test and demonstrate the feature source plugin** + Jdbc { + driver = org.postgresql.Driver + url = "jdbc:postgresql://flink_e2e_greenplum:5432/testdb" + user = tester + password = pivotal + query = "select age, name from source" + } - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Jdbc } transform { - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { - Jdbc { - driver = org.postgresql.Driver - url = "jdbc:postgresql://flink_e2e_greenplum:5432/testdb" - user = tester - password = pivotal - query = "insert into sink(age, name) values(?, ?)" - } + Jdbc { + driver = org.postgresql.Driver + url = "jdbc:postgresql://flink_e2e_greenplum:5432/testdb" + user = tester + password = pivotal + query = "insert into sink(age, name) values(?, ?)" + } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf index c038057b823..2e4b788606a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/jdbc_phoenix_source_and_sink.conf @@ -28,11 +28,11 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - Jdbc { - driver = org.apache.phoenix.queryserver.client.Driver - url = "jdbc:phoenix:thin:url=http://seatunnel_e2e_phoenix:8765;serialization=PROTOBUF" - query = "select * from test.SOURCE" - } + Jdbc { + driver = org.apache.phoenix.queryserver.client.Driver + url = "jdbc:phoenix:thin:url=http://seatunnel_e2e_phoenix:8765;serialization=PROTOBUF" + query = "select * from test.SOURCE" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource @@ -46,11 +46,11 @@ transform { } sink { - Jdbc { - driver = org.apache.phoenix.queryserver.client.Driver - url = "jdbc:phoenix:thin:url=http://seatunnel_e2e_phoenix:8765;serialization=PROTOBUF" - query = "upsert into test.SINK(age, name) values(?, ?)" - } + Jdbc { + driver = org.apache.phoenix.queryserver.client.Driver + url = "jdbc:phoenix:thin:url=http://seatunnel_e2e_phoenix:8765;serialization=PROTOBUF" + query = "upsert into test.SINK(age, name) values(?, ?)" + } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/selectdb-jdbc-to-selectdb.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/selectdb-jdbc-to-selectdb.conf index 87c166cee34..9795d4132ed 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/selectdb-jdbc-to-selectdb.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-2/src/test/resources/selectdb-jdbc-to-selectdb.conf @@ -16,8 +16,8 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { @@ -34,16 +34,16 @@ transform { } sink { - SelectDBCloud { - load-url = "selectdb_e2e:8030" - jdbc-url = "selectdb_e2e:9030" - username = "admin" - password = "" - cluster-name = "cluster" - table.identifier = "test.e2e_table_sink" - doris.config = { - file.type="json" - file.strip_outer_array="false" - } + SelectDBCloud { + load-url = "selectdb_e2e:8030" + jdbc-url = "selectdb_e2e:9030" + username = "admin" + password = "" + cluster-name = "cluster" + table.identifier = "test.e2e_table_sink" + doris.config = { + file.type = "json" + file.strip_outer_array = "false" } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/extractTopic_fake_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/extractTopic_fake_to_kafka.conf index 59bd7e9e894..356c41be41f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/extractTopic_fake_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/extractTopic_fake_to_kafka.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -74,6 +74,6 @@ sink { bootstrap.servers = "kafkaCluster:9092" topic = "${c_string}" format = json - partition_key_fields = ["c_map","c_string"] + partition_key_fields = ["c_map", "c_string"] } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkaTextsink_fake_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkaTextsink_fake_to_kafka.conf index 71a2605ea13..17ec327b302 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkaTextsink_fake_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkaTextsink_fake_to_kafka.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -65,6 +65,6 @@ sink { topic = "test_text_topic" format = text field_delimiter = "," - partition_key_fields = ["c_map","c_string"] + partition_key_fields = ["c_map", "c_string"] } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka_default_sink_fake_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka_default_sink_fake_to_kafka.conf index bf82b5e9d3c..a6321487405 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka_default_sink_fake_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafka_default_sink_fake_to_kafka.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasink_fake_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasink_fake_to_kafka.conf index a4ad964d313..fe344a71d04 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasink_fake_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasink_fake_to_kafka.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -64,6 +64,6 @@ sink { bootstrap.servers = "kafkaCluster:9092" topic = "test_topic" format = json - partition_key_fields = ["c_map","c_string"] + partition_key_fields = ["c_map", "c_string"] } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_cdc_to_pgsql.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_cdc_to_pgsql.conf index f5488f4bca6..cd393455fe8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_cdc_to_pgsql.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_cdc_to_pgsql.conf @@ -19,15 +19,15 @@ ###### env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -38,10 +38,10 @@ source { start_mode = earliest schema = { fields { - id = "int" - name = "string" - description = "string" - weight = "string" + id = "int" + name = "string" + description = "string" + weight = "string" } }, format = canal_json @@ -50,14 +50,14 @@ source { sink { - Jdbc { - driver = org.postgresql.Driver - url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" - user = test - password = test - generate_sink_sql = true - database = public - table = sink - primary_keys = ["id"] - } + Jdbc { + driver = org.postgresql.Driver + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" + user = test + password = test + generate_sink_sql = true + database = public + table = sink + primary_keys = ["id"] + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_to_kafka.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_to_kafka.conf index 44f51789b63..c229c88f9fc 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_to_kafka.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-kafka-e2e/src/test/resources/kafkasource_canal_to_kafka.conf @@ -19,15 +19,15 @@ ###### env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -39,10 +39,10 @@ source { format = canal_json schema = { fields { - id = "int" - name = "string" - description = "string" - weight = "string" + id = "int" + name = "string" + description = "string" + weight = "string" } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-maxcompute-e2e/src/test/resources/maxcompute_to_fake.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-maxcompute-e2e/src/test/resources/maxcompute_to_fake.conf index 646722435e8..b7a5a5ff6ef 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-maxcompute-e2e/src/test/resources/maxcompute_to_fake.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-maxcompute-e2e/src/test/resources/maxcompute_to_fake.conf @@ -28,11 +28,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** Maxcompute { parallelism = 3 - accessId="xxx" - accesskey="xxx" - endpoint="xxx" - project="xxx" - table_name="xxx" + accessId = "xxx" + accesskey = "xxx" + endpoint = "xxx" + project = "xxx" + table_name = "xxx" #partition_spec="ds='20220101'" split_row = 3 } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf index 21ee6f69f6b..6aa62797309 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf @@ -30,34 +30,34 @@ env { } source { - Neo4j { - uri = "neo4j://neo4j-host:7687" - username = "neo4j" - password = "Test@12343" - database = "neo4j" + Neo4j { + uri = "neo4j://neo4j-host:7687" + username = "neo4j" + password = "Test@12343" + database = "neo4j" - max_transaction_retry_time = 1 - max_connection_timeout = 1 + max_transaction_retry_time = 1 + max_connection_timeout = 1 - query = "MATCH (t:Test) WITH *, t{.int} AS _map RETURN t.string, t.boolean, t.long, t.double, t.byteArray, t.date, t.localDateTime, _map, t.list, t.int, t.float" + query = "MATCH (t:Test) WITH *, t{.int} AS _map RETURN t.string, t.boolean, t.long, t.double, t.byteArray, t.date, t.localDateTime, _map, t.list, t.int, t.float" - schema { - fields { - t.string=STRING - t.boolean=BOOLEAN - t.long=BIGINT - t.double=DOUBLE - t.null=NULL - t.byteArray=BYTES - t.date=DATE - t.localDateTime=TIMESTAMP - _map="MAP" - t.list="ARRAY" - t.int=INT - t.float=FLOAT - } - } + schema { + fields { + t.string = STRING + t.boolean = BOOLEAN + t.long = BIGINT + t.double = DOUBLE + t.null = NULL + t.byteArray = BYTES + t.date = DATE + t.localDateTime = TIMESTAMP + _map = "MAP" + t.list = "ARRAY" + t.int = INT + t.float = FLOAT + } } + } } transform { @@ -65,27 +65,27 @@ transform { sink { Neo4j { - uri = "neo4j://neo4j-host:7687" - username = "neo4j" - password = "Test@12343" - database = "neo4j" + uri = "neo4j://neo4j-host:7687" + username = "neo4j" + password = "Test@12343" + database = "neo4j" - max_transaction_retry_time = 1 - max_connection_timeout = 1 + max_transaction_retry_time = 1 + max_connection_timeout = 1 - query = "CREATE (tt:TestTest {string:$string, boolean:$boolean, long:$long, double:$double, byteArray:$byteArray, date:date($date), localDateTime:localdatetime($localDateTime), list:$list, int:$int, float:$float, mapValue:$map['int']})" - queryParamPosition = { - string=0 - boolean=1 - long=2 - double=3 - byteArray=5 - date=6 - localDateTime=7 - map=8 - list=9 - int=10 - float=11 - } + query = "CREATE (tt:TestTest {string:$string, boolean:$boolean, long:$long, double:$double, byteArray:$byteArray, date:date($date), localDateTime:localdatetime($localDateTime), list:$list, int:$int, float:$float, mapValue:$map['int']})" + queryParamPosition = { + string = 0 + boolean = 1 + long = 2 + double = 3 + byteArray = 5 + date = 6 + localDateTime = 7 + map = 8 + list = 9 + int = 10 + float = 11 + } } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf index 79494c8baf2..0b7480a3ca8 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -16,51 +16,51 @@ # env { - execution.parallelism = 1 - job.mode = "STREAMING" + execution.parallelism = 1 + job.mode = "STREAMING" } source { - RabbitMQ { - host = "rabbitmq-e2e" - port = 5672 - virtual_host = "/" - username = "guest" - password = "guest" - queue_name = "test" - for_e2e_testing = true - schema = { - fields { - id = bigint - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(2, 1)" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } - } + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test" + for_e2e_testing = true + schema = { + fields { + id = bigint + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + } } + } } transform { } sink { - RabbitMQ { - host = "rabbitmq-e2e" - port = 5672 - virtual_host = "/" - username = "guest" - password = "guest" - queue_name = "test1" - } + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test1" + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf index 5cd26838a39..c37008298c0 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/redis-to-redis.conf @@ -16,34 +16,34 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" - shade.identifier = "base64" + execution.parallelism = 1 + job.mode = "BATCH" + shade.identifier = "base64" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - keys = "key_test*" - data_type = key - } + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + keys = "key_test*" + data_type = key + } } sink { - Redis { - host = "redis-e2e" - port = 6379 - auth = "U2VhVHVubmVs" - key = "key_list" - data_type = list - } + Redis { + host = "redis-e2e" + port = 6379 + auth = "U2VhVHVubmVs" + key = "key_list" + data_type = list + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf index ca21a11cea6..305d3b5ec71 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-sink_fake_to_rocketmq.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -63,7 +63,7 @@ sink { Rocketmq { name.srv.addr = "rocketmq-e2e:9876" topic = "test_topic" - partition.key.fields = ["c_map","c_string"] + partition.key.fields = ["c_map", "c_string"] producer.send.sync = true } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf index 935936dc381..e2666b59f1f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rocketmq-e2e/src/test/resources/rocketmq-text-sink_fake_to_rocketmq.conf @@ -16,15 +16,15 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" - #spark config - spark.app.name = "SeaTunnel" - spark.executor.instances = 1 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local + #spark config + spark.app.name = "SeaTunnel" + spark.executor.instances = 1 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local } source { @@ -64,7 +64,7 @@ sink { name.srv.addr = "rocketmq-e2e:9876" topic = "test_text_topic" format = text - partition.key.fields = ["c_map","c_string"] + partition.key.fields = ["c_map", "c_string"] producer.send.sync = true } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-thrift-to-starrocks-streamload.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-thrift-to-starrocks-streamload.conf index 3c7515618f2..d3ca3602fb2 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-thrift-to-starrocks-streamload.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/starrocks-thrift-to-starrocks-streamload.conf @@ -16,8 +16,8 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { @@ -29,22 +29,22 @@ source { table = "e2e_table_source" max_retries = 3 schema { - fields { - BIGINT_COL = BIGINT - LARGEINT_COL = STRING - SMALLINT_COL = SMALLINT - TINYINT_COL = TINYINT - BOOLEAN_COL = BOOLEAN - DECIMAL_COL = "DECIMAL(20, 1)" - DOUBLE_COL = DOUBLE - FLOAT_COL = FLOAT - INT_COL = INT - CHAR_COL = STRING - VARCHAR_11_COL = STRING - STRING_COL = STRING - DATETIME_COL = TIMESTAMP - DATE_COL = DATE - } + fields { + BIGINT_COL = BIGINT + LARGEINT_COL = STRING + SMALLINT_COL = SMALLINT + TINYINT_COL = TINYINT + BOOLEAN_COL = BOOLEAN + DECIMAL_COL = "DECIMAL(20, 1)" + DOUBLE_COL = DOUBLE + FLOAT_COL = FLOAT + INT_COL = INT + CHAR_COL = STRING + VARCHAR_11_COL = STRING + STRING_COL = STRING + DATETIME_COL = TIMESTAMP + DATE_COL = DATE + } } } } @@ -53,18 +53,18 @@ transform { } sink { - StarRocks { - nodeUrls = ["starrocks_e2e:8030"] - username = root - password = "" - database = "test" - table = "e2e_table_sink" - batch_max_rows = 100 - max_retries = 3 + StarRocks { + nodeUrls = ["starrocks_e2e:8030"] + username = root + password = "" + database = "test" + table = "e2e_table_sink" + batch_max_rows = 100 + max_retries = 3 - starrocks.config = { - format = "JSON" - strip_outer_array = true - } + starrocks.config = { + format = "JSON" + strip_outer_array = true } + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/write-cdc-changelog-to-starrocks.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/write-cdc-changelog-to-starrocks.conf index 1a4066d7020..fe8f9829a63 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/write-cdc-changelog-to-starrocks.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-starrocks-e2e/src/test/resources/write-cdc-changelog-to-starrocks.conf @@ -16,63 +16,63 @@ # env { - execution.parallelism = 1 - job.mode = "BATCH" + execution.parallelism = 1 + job.mode = "BATCH" } source { - FakeSource { - schema = { - fields { - pk_id = bigint - name = string - score = int - } - } - rows = [ - { - kind = INSERT - fields = [1, "A", 100] - }, - { - kind = INSERT - fields = [2, "B", 100] - }, - { - kind = INSERT - fields = [3, "C", 100] - }, - { - kind = UPDATE_BEFORE - fields = [1, "A", 100] - }, - { - kind = UPDATE_AFTER - fields = [1, "A_1", 100] - }, - { - kind = DELETE - fields = [2, "B", 100] - } - ] + FakeSource { + schema = { + fields { + pk_id = bigint + name = string + score = int + } } + rows = [ + { + kind = INSERT + fields = [1, "A", 100] + }, + { + kind = INSERT + fields = [2, "B", 100] + }, + { + kind = INSERT + fields = [3, "C", 100] + }, + { + kind = UPDATE_BEFORE + fields = [1, "A", 100] + }, + { + kind = UPDATE_AFTER + fields = [1, "A_1", 100] + }, + { + kind = DELETE + fields = [2, "B", 100] + } + ] + } } sink { - StarRocks { - nodeUrls = ["starrocks_cdc_e2e:8030"] - username = root - password = "" - database = "test" - table = "e2e_table_sink" + StarRocks { + nodeUrls = ["starrocks_cdc_e2e:8030"] + username = root + password = "" + database = "test" + table = "e2e_table_sink" - batch_max_rows = 100 - max_retries = 3 + batch_max_rows = 100 + max_retries = 3 - starrocks.config = { - format = "CSV" - } - - enable_upsert_delete = true + starrocks.config = { + format = "CSV" } + + enable_upsert_delete = true + } } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf index 39b001ea042..057c65ee2ba 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-tdengine-e2e/src/test/resources/tdengine/tdengine_source_to_sink.conf @@ -28,16 +28,16 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - TDengine { - url : "jdbc:TAOS-RS://flink_e2e_tdengine_src:6041/" - username : "root" - password : "taosdata" - database : "power" - stable : "meters" - lower_bound : "2018-10-03 14:38:05.000" - upper_bound : "2018-10-03 14:38:16.801" - result_table_name = "tdengine_result" - } + TDengine { + url: "jdbc:TAOS-RS://flink_e2e_tdengine_src:6041/" + username: "root" + password: "taosdata" + database: "power" + stable: "meters" + lower_bound: "2018-10-03 14:38:05.000" + upper_bound: "2018-10-03 14:38:16.801" + result_table_name = "tdengine_result" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/category/source-v2 } @@ -46,14 +46,14 @@ transform { } sink { - TDengine { - url : "jdbc:TAOS-RS://flink_e2e_tdengine_sink:6041/" - username : "root" - password : "taosdata" - database : "power2" - stable : "meters2" - timezone : "UTC" - } + TDengine { + url: "jdbc:TAOS-RS://flink_e2e_tdengine_sink:6041/" + username: "root" + password: "taosdata" + database: "power2" + stable: "meters2" + timezone: "UTC" + } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, # please go to https://seatunnel.apache.org/docs/category/sink-v2 } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf index ffc827d8f86..42df3525d8e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-console-seatunnel-e2e/src/test/resources/fakesource_to_console.conf @@ -24,16 +24,16 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - schema { - fields { - id = "int" - name = "string" - age = "int" - } + FakeSource { + result_table_name = "fake" + schema { + fields { + id = "int" + name = "string" + age = "int" } } + } } sink { diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf index 78ea7db948e..bc61e06312e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file.conf @@ -27,17 +27,33 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = 1 - result_table_name = "fake" - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = 1 + result_table_name = "fake" + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -52,26 +68,10 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } } transform { @@ -79,16 +79,16 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test1" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["c_string"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test1" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_string"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf index 3d6ca5054b1..f6fda45b1ff 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/batch_fakesource_to_file_complex.conf @@ -27,17 +27,33 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = 1 - result_table_name = "fake" - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = 1 + result_table_name = "fake" + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -52,38 +68,38 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - result_table_name = "fake2" - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + result_table_name = "fake2" + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -98,26 +114,10 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } } transform { @@ -125,17 +125,17 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["c_string"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error", - source_table_name=["fake","fake2"] + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_string"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error", + source_table_name = ["fake", "fake2"] } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_template.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_template.conf index f469c961fc2..4c2015a865a 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_template.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_template.conf @@ -27,18 +27,34 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = ${dynamic_test_row_num_per_parallelism} - split.num = 5 - split.read-interval = 3000 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = ${dynamic_test_parallelism} - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = ${dynamic_test_row_num_per_parallelism} + split.num = 5 + split.read-interval = 3000 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = ${dynamic_test_parallelism} + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -53,26 +69,10 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } } transform { @@ -80,13 +80,13 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run - field_delimiter="\t" - row_delimiter="\n" - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run + field_delimiter = "\t" + row_delimiter = "\n" + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_two_pipeline_template.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_two_pipeline_template.conf index 4d7f65d53f4..7e01c01c984 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_two_pipeline_template.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/cluster_batch_fake_to_localfile_two_pipeline_template.conf @@ -27,19 +27,35 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = table1 - row.num = ${dynamic_test_row_num_per_parallelism} - split.num = 5 - split.read-interval = 3000 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = ${dynamic_test_parallelism} - schema = { - fields { - c_map = "map>" + FakeSource { + result_table_name = table1 + row.num = ${dynamic_test_row_num_per_parallelism} + split.num = 5 + split.read-interval = 3000 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = ${dynamic_test_parallelism} + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -54,74 +70,58 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } - FakeSource { - result_table_name = table2 - row.num = ${dynamic_test_row_num_per_parallelism} - split.num = 5 - split.read-interval = 3000 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = ${dynamic_test_parallelism} - schema = { - fields { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } - } - } + FakeSource { + result_table_name = table2 + row.num = ${dynamic_test_row_num_per_parallelism} + split.num = 5 + split.read-interval = 3000 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = ${dynamic_test_parallelism} + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp } + } + } + } } transform { @@ -130,25 +130,25 @@ transform { sink { LocalFile { source_table_name = table1 - path="/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run - field_delimiter="\t" - row_delimiter="\n" - file_name_expression="${transactionId}_${now}" - file_format="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" - } + path = "/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run + field_delimiter = "\t" + row_delimiter = "\n" + file_name_expression = "${transactionId}_${now}" + file_format = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" + } LocalFile { source_table_name = table2 - path="/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run - field_delimiter="\t" - row_delimiter="\n" - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" - } + path = "/tmp/hive/warehouse/${dynamic_test_case_name}" # dynamic_test_case_name will be replace to the final file name before test run + field_delimiter = "\t" + row_delimiter = "\n" + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" + } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/stream_fakesource_to_file.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/stream_fakesource_to_file.conf index c8b3867301f..ab5c81c24be 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/stream_fakesource_to_file.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/stream_fakesource_to_file.conf @@ -27,17 +27,33 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - parallelism = 1 - result_table_name = "fake" - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + parallelism = 1 + result_table_name = "fake" + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -52,26 +68,10 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } } transform { @@ -79,16 +79,16 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test1" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["c_string"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test1" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_string"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf index 15eb0c4292a..1333cabd65c 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/streaming_fakesource_to_file_complex.conf @@ -27,17 +27,33 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - result_table_name = "fake" - parallelism = 1 - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + result_table_name = "fake" + parallelism = 1 + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -52,38 +68,38 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } - FakeSource { - row.num = 10 - map.size = 10 - array.size = 10 - bytes.length = 10 - string.length = 10 - result_table_name = "fake2" - parallelism = 1 - schema = { - fields { - c_map = "map>" + FakeSource { + row.num = 10 + map.size = 10 + array.size = 10 + bytes.length = 10 + string.length = 10 + result_table_name = "fake2" + parallelism = 1 + schema = { + fields { + c_map = "map>" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + c_row = { + c_map = "map>" c_array = "array" c_string = string c_boolean = boolean @@ -98,26 +114,10 @@ source { c_bytes = bytes c_date = date c_timestamp = timestamp - c_row = { - c_map = "map>" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } } } } + } } transform { @@ -125,17 +125,17 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test3" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["c_string"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format_type="text" - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error", - source_table_name=["fake","fake2"] + path = "/tmp/hive/warehouse/test3" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_string"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format_type = "text" + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error", + source_table_name = ["fake", "fake2"] } } diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/field_mapper_transform.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/field_mapper_transform.conf index 2071e66d5ca..c2d1f225f2b 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/field_mapper_transform.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/field_mapper_transform.conf @@ -44,10 +44,10 @@ transform { source_table_name = "fake" result_table_name = "fake1" field_mapper = { - id = id - age = age_as - int1 = int1_as - name = name + id = id + age = age_as + int1 = int1_as + name = name } } } @@ -80,19 +80,19 @@ sink { field_name = age_as field_type = int field_value = [ - { - rule_type = NOT_NULL - } + { + rule_type = NOT_NULL + } ] }, { - field_name = int1_as - field_type = int - field_value = [ - { - rule_type = NOT_NULL - } - ] + field_name = int1_as + field_type = int + field_value = [ + { + rule_type = NOT_NULL + } + ] }, { field_name = name diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/binary_expression.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/binary_expression.conf index 595e9ea5f83..22ab4cc31eb 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/binary_expression.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/binary_expression.conf @@ -35,7 +35,7 @@ source { } } rows = [ - { fields = [1, "Joy Ding", 134.22], kind = INSERT } + {fields = [1, "Joy Ding", 134.22], kind = INSERT} ] } } @@ -60,42 +60,42 @@ sink { field_name = "id" field_type = "int" field_value = [ - { equals_to = 2 } + {equals_to = 2} ] }, { field_name = "id2" field_type = "int" field_value = [ - { equals_to = 4 } + {equals_to = 4} ] }, { field_name = "price" field_type = "double" field_value = [ - { equals_to = 44.74 } + {equals_to = 44.74} ] }, { field_name = "price2" field_type = "double" field_value = [ - { equals_to = 100 } + {equals_to = 100} ] }, { field_name = "price3" field_type = "double" field_value = [ - { equals_to = 18.619999999999994 } + {equals_to = 18.619999999999994} ] }, { field_name = "name" field_type = "string" field_value = [ - { equals_to = "Joy Ding_1" } + {equals_to = "Joy Ding_1"} ] } ] diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/criteria_filter.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/criteria_filter.conf index 18b5fe596d6..a8c51a87497 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/criteria_filter.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/criteria_filter.conf @@ -36,9 +36,9 @@ source { } } rows = [ - { fields = [1, "Joy Ding", 20, null], kind = INSERT } - { fields = [2, "May Ding", 22, "may_ding@apache.com"], kind = INSERT } - { fields = [3, "Kin Dom", 21, "kin_dom@apache.com"], kind = INSERT } + {fields = [1, "Joy Ding", 20, null], kind = INSERT} + {fields = [2, "May Ding", 22, "may_ding@apache.com"], kind = INSERT} + {fields = [3, "Kin Dom", 21, "kin_dom@apache.com"], kind = INSERT} ] } } diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_datetime.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_datetime.conf index 9a977609aaf..936094351c3 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_datetime.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_datetime.conf @@ -40,7 +40,7 @@ source { } } rows = [ - { fields = [1, "Joy Ding", "2021-04-15T13:34:45", "2022-01-23T12:34:56", "2021-04-15T13:34:45.235", "2021-04-08T13:34:45.235", "2021-04-08 13:34:45.235", "2021-04-08"], kind = INSERT } + {fields = [1, "Joy Ding", "2021-04-15T13:34:45", "2022-01-23T12:34:56", "2021-04-15T13:34:45.235", "2021-04-08T13:34:45.235", "2021-04-08 13:34:45.235", "2021-04-08"], kind = INSERT} ] } } @@ -65,161 +65,161 @@ sink { field_name = "cd" field_type = "date" field_value = [ - { rule_type = NOT_NULL } + {rule_type = NOT_NULL} ] }, { field_name = "ct2" field_type = "timestamp" field_value = [ - { rule_type = NOT_NULL } + {rule_type = NOT_NULL} ] }, { field_name = "c1_1" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-16T13:34:45" } + {equals_to = "2021-04-16T13:34:45"} ] }, { field_name = "c1_2" field_type = "timestamp" field_value = [ - { equals_to = "2021-05-25T13:34:45" } + {equals_to = "2021-05-25T13:34:45"} ] }, { field_name = "c1_3" field_type = "timestamp" field_value = [ - { equals_to = "2023-04-15T13:34:45" } + {equals_to = "2023-04-15T13:34:45"} ] }, { field_name = "c1_4" field_type = "timestamp" field_value = [ - { equals_to = "2022-02-15T13:34:45" } + {equals_to = "2022-02-15T13:34:45"} ] }, { field_name = "c1_5" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-16T02:34:45" } + {equals_to = "2021-04-16T02:34:45"} ] }, { field_name = "c1_6" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T14:14:45" } + {equals_to = "2021-04-15T14:14:45"} ] }, { field_name = "c1_7" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T13:35:15" } + {equals_to = "2021-04-15T13:35:15"} ] }, { field_name = "test" field_type = "long" field_value = [ - { equals_to = 283 } + {equals_to = 283} ] }, { field_name = "c2_1" field_type = "long" field_value = [ - { equals_to = 283 } + {equals_to = 283} ] }, { field_name = "c2_2" field_type = "long" field_value = [ - { equals_to = 0 } + {equals_to = 0} ] }, { field_name = "c2_3" field_type = "long" field_value = [ - { equals_to = 9 } + {equals_to = 9} ] }, { field_name = "c2_4" field_type = "long" field_value = [ - { equals_to = 6791 } + {equals_to = 6791} ] }, { field_name = "c2_5" field_type = "long" field_value = [ - { equals_to = 407460 } + {equals_to = 407460} ] }, { field_name = "c2_6" field_type = "long" field_value = [ - { equals_to = 24447611 } + {equals_to = 24447611} ] }, { field_name = "c3_1" field_type = "timestamp" field_value = [ - { equals_to = "2021-01-01T00:00:00" } + {equals_to = "2021-01-01T00:00:00"} ] }, { field_name = "c3_2" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-01T00:00:00" } + {equals_to = "2021-04-01T00:00:00"} ] }, { field_name = "c3_3" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T00:00:00" } + {equals_to = "2021-04-15T00:00:00"} ] }, { field_name = "c3_4" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T13:00:00" } + {equals_to = "2021-04-15T13:00:00"} ] }, { field_name = "c3_5" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T13:34:00" } + {equals_to = "2021-04-15T13:34:00"} ] }, { field_name = "c3_6" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-15T13:34:45" } + {equals_to = "2021-04-15T13:34:45"} ] }, { field_name = "c3_7" field_type = "string" field_value = [ - { equals_to = "Thursday" } + {equals_to = "Thursday"} ] }, { @@ -227,168 +227,168 @@ sink { field_type = "int" field_value = [ # Thursday - { equals_to = 4 } + {equals_to = 4} ] }, { field_name = "c3_9" field_type = "int" field_value = [ - { equals_to = 105 } + {equals_to = 105} ] }, { field_name = "c3_10" field_type = "int" field_value = [ - { equals_to = 2021 } + {equals_to = 2021} ] }, { field_name = "c3_11" field_type = "int" field_value = [ - { equals_to = 4 } + {equals_to = 4} ] }, { field_name = "c3_12" field_type = "int" field_value = [ - { equals_to = 15 } + {equals_to = 15} ] }, { field_name = "c3_13" field_type = "int" field_value = [ - { equals_to = 13 } + {equals_to = 13} ] }, { field_name = "c3_14" field_type = "int" field_value = [ - { equals_to = 34 } + {equals_to = 34} ] }, { field_name = "c3_15" field_type = "int" field_value = [ - { equals_to = 45 } + {equals_to = 45} ] }, { field_name = "c3_16" field_type = "int" field_value = [ - { equals_to = 235 } + {equals_to = 235} ] }, { field_name = "c3_17" field_type = "int" field_value = [ - { equals_to = 4 } + {equals_to = 4} ] }, { field_name = "c3_18" field_type = "int" field_value = [ - { equals_to = 105 } + {equals_to = 105} ] }, { field_name = "c4_1" field_type = "string" field_value = [ - { equals_to = "2021-04-08 13:34:45.2" } + {equals_to = "2021-04-08 13:34:45.2"} ] }, { field_name = "c4_2" field_type = "string" field_value = [ - { equals_to = "2021-04-08" } + {equals_to = "2021-04-08"} ] }, { field_name = "c4_3" field_type = "string" field_value = [ - { equals_to = "13:34:45.235" } + {equals_to = "13:34:45.235"} ] }, { field_name = "c4_4" field_type = "int" field_value = [ - { equals_to = 13 } + {equals_to = 13} ] }, { field_name = "c4_5" field_type = "int" field_value = [ - { equals_to = 34 } + {equals_to = 34} ] }, { field_name = "c4_6" field_type = "int" field_value = [ - { equals_to = 4 } + {equals_to = 4} ] }, { field_name = "c4_7" field_type = "string" field_value = [ - { equals_to = "April" } + {equals_to = "April"} ] }, { field_name = "c5_1" field_type = "timestamp" field_value = [ - { equals_to = "2021-04-08T13:34:45.235" } + {equals_to = "2021-04-08T13:34:45.235"} ] }, { field_name = "c6_1" field_type = "date" field_value = [ - { equals_to = "2021-04-08" } + {equals_to = "2021-04-08"} ] }, { field_name = "c4_8" field_type = "int" field_value = [ - { equals_to = 2 } + {equals_to = 2} ] }, { field_name = "c4_9" field_type = "int" field_value = [ - { equals_to = 45 } + {equals_to = 45} ] }, { field_name = "c4_10" field_type = "int" field_value = [ - { equals_to = 15 } + {equals_to = 15} ] }, { field_name = "c4_11" field_type = "int" field_value = [ - { equals_to = 2021 } + {equals_to = 2021} ] } ] diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_numeric.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_numeric.conf index ed741000cc0..56f89d005ca 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_numeric.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_numeric.conf @@ -44,7 +44,7 @@ source { } } rows = [ - { fields = [1, "Joy Ding", -120.72, 0, 3.1415926, 13, 13.2, 1324.252, 180, 10.24, 120.72124, 2], kind = INSERT } + {fields = [1, "Joy Ding", -120.72, 0, 3.1415926, 13, 13.2, 1324.252, 180, 10.24, 120.72124, 2], kind = INSERT} ] } } @@ -69,175 +69,175 @@ sink { field_name = "c1_1" field_type = "double" field_value = [ - { equals_to = 120.72 } + {equals_to = 120.72} ] }, { field_name = "id1" field_type = "double" field_value = [ - { equals_to = 0 } + {equals_to = 0} ] }, { field_name = "c2_1" field_type = "double" field_value = [ - { equals_to = 0 } + {equals_to = 0} ] }, { field_name = "c2_2" field_type = "double" field_value = [ - { equals_to = 0.0 } + {equals_to = 0.0} ] }, { field_name = "c2_3" field_type = "double" field_value = [ - { equals_to = 1 } + {equals_to = 1} ] }, { field_name = "c2_4" field_type = "double" field_value = [ - { equals_to = 1 } + {equals_to = 1} ] }, { field_name = "c2_5" field_type = "double" field_value = [ - { equals_to = 0.0 } + {equals_to = 0.0} ] }, { field_name = "c2_6" field_type = "double" field_value = [ - { equals_to = 0.0 } + {equals_to = 0.0} ] }, { field_name = "c3_1" field_type = "double" field_value = [ - { equals_to = 0.9999999732051038 } + {equals_to = 0.9999999732051038} ] }, { field_name = "c2_7" field_type = "double" field_value = [ - { equals_to = 0.0 } + {equals_to = 0.0} ] }, { field_name = "c4_1" field_type = "int" field_value = [ - { equals_to = 3 } + {equals_to = 3} ] }, { field_name = "c4_2" field_type = "double" field_value = [ - { equals_to = 2.2 } + {equals_to = 2.2} ] }, { field_name = "c5_1" field_type = "int" field_value = [ - { equals_to = 14 } + {equals_to = 14} ] }, { field_name = "c10_1" field_type = "double" field_value = [ - { equals_to = 7.38905609893065 } + {equals_to = 7.38905609893065} ] }, { field_name = "c5_2" field_type = "int" field_value = [ - { equals_to = 13 } + {equals_to = 13} ] }, { field_name = "c5_3" field_type = "double" field_value = [ - { equals_to = 2.580216829592325 } + {equals_to = 2.580216829592325} ] }, { field_name = "c5_4" field_type = "double" field_value = [ - { equals_to = 1.1205739312058498 } + {equals_to = 1.1205739312058498} ] }, { field_name = "c6_1" field_type = "double" field_value = [ - { equals_to = 3.1219706375172507 } + {equals_to = 3.1219706375172507} ] }, { field_name = "c7_1" field_type = "double" field_value = [ - { equals_to = 3.141592653589793 } + {equals_to = 3.141592653589793} ] }, { field_name = "c8_1" field_type = "double" field_value = [ - { equals_to = 3.2 } + {equals_to = 3.2} ] }, { field_name = "pi" field_type = "double" field_value = [ - { equals_to = 3.141592653589793 } + {equals_to = 3.141592653589793} ] }, { field_name = "c5_5" field_type = "double" field_value = [ - { equals_to = 174.23999999999998 } + {equals_to = 174.23999999999998} ] }, { field_name = "c9_1" field_type = "double" field_value = [ - { equals_to = 120.72 } + {equals_to = 120.72} ] }, { field_name = "c1_2" field_type = "int" field_value = [ - { equals_to = -1 } + {equals_to = -1} ] }, { field_name = "c9_2" field_type = "double" field_value = [ - { equals_to = 120.72 } + {equals_to = 120.72} ] } ] diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf index c752ff37b23..806abb8fb66 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_string.conf @@ -45,7 +45,7 @@ source { } } rows = [ - { fields = [1, "Joy Ding", "A", "b" , "&", "&^^$wef9", 98, "0037", "7", "*Joy_Ding@s.com*", "Joy_WWWDing@s.com", "2020-10-01", "2022-12-12T23:34:45"], kind = INSERT } + {fields = [1, "Joy Ding", "A", "b", "&", "&^^$wef9", 98, "0037", "7", "*Joy_Ding@s.com*", "Joy_WWWDing@s.com", "2020-10-01", "2022-12-12T23:34:45"], kind = INSERT} ] } } @@ -70,224 +70,224 @@ sink { field_name = "c1_1" field_type = "int" field_value = [ - { equals_to = 65 } + {equals_to = 65} ] }, { field_name = "c2_1" field_type = "int" field_value = [ - { equals_to = 98 } + {equals_to = 98} ] }, { field_name = "c4_1" field_type = "long" field_value = [ - { equals_to = 64 } + {equals_to = 64} ] }, { field_name = "c4_2" field_type = "long" field_value = [ - { equals_to = 8 } + {equals_to = 8} ] }, { field_name = "c4_3" field_type = "long" field_value = [ - { equals_to = 8 } + {equals_to = 8} ] }, { field_name = "c5_1" field_type = "string" field_value = [ - { equals_to = "b" } + {equals_to = "b"} ] }, { field_name = "c1_2" field_type = "string" field_value = [ - { equals_to = "A1!" } + {equals_to = "A1!"} ] }, { field_name = "c6_1" field_type = "string" field_value = [ - { equals_to = "7" } + {equals_to = "7"} ] }, { field_name = "c7_1" field_type = "string" field_value = [ - { equals_to = "0037" } + {equals_to = "0037"} ] }, { field_name = "name1" field_type = "string" field_value = [ - { equals_to = "J** Ding" } + {equals_to = "J** Ding"} ] }, { field_name = "name2" field_type = "string" field_value = [ - { equals_to = "joy ding" } + {equals_to = "joy ding"} ] }, { field_name = "name3" field_type = "string" field_value = [ - { equals_to = "JOY DING" } + {equals_to = "JOY DING"} ] }, { field_name = "name4" field_type = "string" field_value = [ - { equals_to = "Joy" } + {equals_to = "Joy"} ] }, { field_name = "name5" field_type = "string" field_value = [ - { equals_to = "Ding" } + {equals_to = "Ding"} ] }, { field_name = "name6" field_type = "string" field_value = [ - { equals_to = "**Joy Ding" } + {equals_to = "**Joy Ding"} ] }, { field_name = "name7" field_type = "string" field_value = [ - { equals_to = "Joy Ding**" } + {equals_to = "Joy Ding**"} ] }, { field_name = "c8_1" field_type = "string" field_value = [ - { equals_to = "Joy_Ding@s.com*" } + {equals_to = "Joy_Ding@s.com*"} ] }, { field_name = "c8_2" field_type = "string" field_value = [ - { equals_to = "*Joy_Ding@s.com" } + {equals_to = "*Joy_Ding@s.com"} ] }, { field_name = "c8_3" field_type = "string" field_value = [ - { equals_to = "Joy_Ding@s.com" } + {equals_to = "Joy_Ding@s.com"} ] }, { field_name = "c9_1" field_type = "string" field_value = [ - { equals_to = "Joy_WDing@s.com" } + {equals_to = "Joy_WDing@s.com"} ] }, { field_name = "name8" field_type = "boolean" field_value = [ - { equals_to = true } + {equals_to = true} ] }, { field_name = "c10_1" field_type = "string" field_value = [ - { equals_to = "2020" } + {equals_to = "2020"} ] }, { field_name = "c10_2" field_type = "string" field_value = [ - { equals_to = "10" } + {equals_to = "10"} ] }, { field_name = "name9" field_type = "string" field_value = [ - { equals_to = "Joy Ding Joy Ding Joy Ding " } + {equals_to = "Joy Ding Joy Ding Joy Ding "} ] }, { field_name = "name10" field_type = "string" field_value = [ - { equals_to = "Joy_Ding" } + {equals_to = "Joy_Ding"} ] }, { field_name = "name11" field_type = "string" field_value = [ - { equals_to = "J352" } + {equals_to = "J352"} ] }, { field_name = "name12" field_type = "string" field_value = [ - { equals_to = "Joy Ding " } + {equals_to = "Joy Ding "} ] }, { field_name = "name13" field_type = "string" field_value = [ - { equals_to = "Joy" } + {equals_to = "Joy"} ] }, { field_name = "id1" field_type = "string" field_value = [ - { equals_to = "1" } + {equals_to = "1"} ] }, { field_name = "c11_1" field_type = "string" field_value = [ - { equals_to = "2022-12-12" } + {equals_to = "2022-12-12"} ] }, { field_name = "name14" field_type = "string" field_value = [ - { equals_to = "Joy DING" } + {equals_to = "Joy DING"} ] }, { field_name = "name15" field_type = "string" field_value = [ - { equals_to = "Joy Ding" } + {equals_to = "Joy Ding"} ] } ] diff --git a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf index 05cc0460377..c28849e4406 100644 --- a/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf +++ b/seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/resources/sql_transform/func_system.conf @@ -37,7 +37,7 @@ source { } } rows = [ - { fields = [1, "Joy Ding", "12.4", "2012-12-21T12:34:56", null], kind = INSERT } + {fields = [1, "Joy Ding", "12.4", "2012-12-21T12:34:56", null], kind = INSERT} ] } } @@ -62,63 +62,63 @@ sink { field_name = "id" field_type = "string" field_value = [ - { equals_to = "1" } + {equals_to = "1"} ] }, { field_name = "id2" field_type = "int" field_value = [ - { equals_to = 1 } + {equals_to = 1} ] }, { field_name = "id3" field_type = "double" field_value = [ - { equals_to = 1 } + {equals_to = 1} ] }, { field_name = "c1_1" field_type = "double" field_value = [ - { equals_to = 12.4 } + {equals_to = 12.4} ] }, { field_name = "c2_1" field_type = "date" field_value = [ - { equals_to = "2012-12-21" } + {equals_to = "2012-12-21"} ] }, { field_name = "c3_1" field_type = "string" field_value = [ - { equals_to = "Unknown" } + {equals_to = "Unknown"} ] }, { field_name = "c3_2" field_type = "string" field_value = [ - { equals_to = "Unknown" } + {equals_to = "Unknown"} ] }, { field_name = "name1" field_type = "string" field_value = [ - { equals_to = "NULL" } + {equals_to = "NULL"} ] }, { field_name = "name2" field_type = "string" field_value = [ - { equals_to = "Joy Ding" } + {equals_to = "Joy Ding"} ] } ] diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file.conf b/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file.conf index 5b57bdbc458..4f5cffa4a92 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file.conf +++ b/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file.conf @@ -45,18 +45,18 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["age"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["name", "age"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } } \ No newline at end of file diff --git a/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file_complex.conf b/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file_complex.conf index f4acfb7a5f2..c687fc1cf24 100644 --- a/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file_complex.conf +++ b/seatunnel-engine/seatunnel-engine-client/src/test/resources/batch_fakesource_to_file_complex.conf @@ -56,18 +56,18 @@ transform { sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error", - source_table_name=["fake","fake2"] + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["age"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["name", "age"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error", + source_table_name = ["fake", "fake2"] } } \ No newline at end of file