From 09ed768009614470d2eba90f7bf5a74bb64e1295 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 13 Aug 2022 22:06:00 +0800 Subject: [PATCH 01/23] [Connector-V2][JDBC-connector] optimization fake --- .../connector-fake/pom.xml | 1 - .../seatunnel/fake/source/FakeData.java | 61 +++++++++++++++++++ .../seatunnel/fake/source/FakeSource.java | 6 +- .../fake/source/FakeSourceReader.java | 15 +---- .../main/resources/examples/spark.batch.conf | 10 +-- .../serialization/RowConverter.java | 4 +- 6 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java diff --git a/seatunnel-connectors-v2/connector-fake/pom.xml b/seatunnel-connectors-v2/connector-fake/pom.xml index 7cb8ed3e4e2..60bb5042e36 100644 --- a/seatunnel-connectors-v2/connector-fake/pom.xml +++ b/seatunnel-connectors-v2/connector-fake/pom.xml @@ -28,7 +28,6 @@ 4.0.0 connector-fake - org.apache.seatunnel diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java new file mode 100644 index 00000000000..0062314d267 --- /dev/null +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -0,0 +1,61 @@ +package org.apache.seatunnel.connectors.seatunnel.fake.source; + +import java.math.BigDecimal; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +/** + * + **/ +public class FakeData { + + public static final String[] columnName = new String[]{ + "c_void", + "c_boolean", + "c_byte", + "c_short", + "c_int", + "c_long", + "c_float", + "c_double", + "c_string", + "c_decimal" + }; + public static final SeaTunnelDataType[] columnType = new SeaTunnelDataType[]{ + BasicType.VOID_TYPE, + BasicType.BOOLEAN_TYPE, + BasicType.BYTE_TYPE, + BasicType.SHORT_TYPE, + BasicType.INT_TYPE, + BasicType.LONG_TYPE, + BasicType.FLOAT_TYPE, + BasicType.DOUBLE_TYPE, + BasicType.STRING_TYPE, + new DecimalType(38, 16) + }; + + + public static SeaTunnelRow generateRow() { + Object[] columnValue = { + Void.TYPE, + RandomUtils.nextInt(0, 2) == 1, + (byte) RandomUtils.nextInt(0, Byte.MAX_VALUE), + (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE), + RandomUtils.nextInt(Short.MAX_VALUE, Integer.MAX_VALUE), + RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE), + RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE), + RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE), + RandomStringUtils.random(10), + BigDecimal.valueOf(RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE)) + }; + if(columnValue.length != columnValue.length || columnValue.length != columnType.length){ + throw new RuntimeException("the row data should be equals to column"); + } + return new SeaTunnelRow(columnValue); + } + +} diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java index 65e6587f47c..18c63dd4f41 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java @@ -20,8 +20,6 @@ import org.apache.seatunnel.api.common.SeaTunnelContext; import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.constants.JobMode; @@ -47,8 +45,8 @@ public Boundedness getBoundedness() { @Override public SeaTunnelRowType getProducedType() { return new SeaTunnelRowType( - new String[]{"name", "age", "timestamp"}, - new SeaTunnelDataType[]{BasicType.STRING_TYPE, BasicType.INT_TYPE, BasicType.LONG_TYPE}); + FakeData.columnName, + FakeData.columnType); } @Override diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java index 7007eaf25b3..f51f9cb9dfa 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java @@ -26,18 +26,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Random; -import java.util.concurrent.ThreadLocalRandom; - public class FakeSourceReader extends AbstractSingleSplitReader { private static final Logger LOGGER = LoggerFactory.getLogger(FakeSourceReader.class); private final SingleSplitReaderContext context; - private final String[] names = {"Wenjun", "Fanjia", "Zongwen", "CalvinKirs"}; - private final int[] ages = {11, 22, 33, 44}; - public FakeSourceReader(SingleSplitReaderContext context) { this.context = context; } @@ -56,13 +50,8 @@ public void close() { @SuppressWarnings("magicnumber") public void pollNext(Collector output) throws InterruptedException { // Generate a random number of rows to emit. - Random random = ThreadLocalRandom.current(); - int size = random.nextInt(10) + 1; - for (int i = 0; i < size; i++) { - int randomIndex = random.nextInt(names.length); - SeaTunnelRow seaTunnelRow = new SeaTunnelRow(new Object[]{names[randomIndex], ages[randomIndex], System.currentTimeMillis()}); - output.collect(seaTunnelRow); - } + SeaTunnelRow seaTunnelRow = FakeData.generateRow(); + output.collect(seaTunnelRow); if (Boundedness.BOUNDED.equals(context.getBoundedness())) { // signal to the source that we have reached the end of the data. LOGGER.info("Closed the bounded fake source"); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index 45ab040b4f1..37be7ca17b8 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -34,7 +34,7 @@ source { # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { result_table_name = "fake" - field_name = "name,age,timestamp" + field_name = "c_void, c_boolean, c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal" } # You can also use other input plugins, such as hdfs @@ -45,7 +45,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/2.1.3/category/source } transform { @@ -53,12 +53,12 @@ transform { # you can also use other transform plugins, such as sql sql { - sql = "select name,age from fake" + sql = "select c_void, c_boolean,c_byte, c_short, c_int, c_long, c_float, c_double, c_string from fake" result_table_name = "sql" } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split + # please go to https://seatunnel.apache.org/docs/2.1.3/category/transform } sink { @@ -72,5 +72,5 @@ sink { # } # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/2.1.3/category/sink } diff --git a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/serialization/RowConverter.java b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/serialization/RowConverter.java index 4b6495e7f84..f642cfe45fb 100644 --- a/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/serialization/RowConverter.java +++ b/seatunnel-translation/seatunnel-translation-base/src/main/java/org/apache/seatunnel/translation/serialization/RowConverter.java @@ -57,12 +57,12 @@ public void validate(SeaTunnelRow seaTunnelRow) throws IOException { } } if (errors.size() > 0) { - throw new UnsupportedOperationException(""); + throw new UnsupportedOperationException(String.join(",", errors)); } } protected boolean validate(Object field, SeaTunnelDataType dataType) { - if (field == null) { + if (field == null || dataType.getSqlType() == SqlType.NULL) { return true; } SqlType sqlType = dataType.getSqlType(); From 7a378e21cc481ea6bf3325eea2d68cfe45cbc25f Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 13 Aug 2022 22:20:21 +0800 Subject: [PATCH 02/23] add more fake data row --- .../connectors/seatunnel/fake/source/FakeSourceReader.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java index f51f9cb9dfa..5b413d81663 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java @@ -50,8 +50,10 @@ public void close() { @SuppressWarnings("magicnumber") public void pollNext(Collector output) throws InterruptedException { // Generate a random number of rows to emit. - SeaTunnelRow seaTunnelRow = FakeData.generateRow(); - output.collect(seaTunnelRow); + for (int i = 0; i < 10; i++) { + SeaTunnelRow seaTunnelRow = FakeData.generateRow(); + output.collect(seaTunnelRow); + } if (Boundedness.BOUNDED.equals(context.getBoundedness())) { // signal to the source that we have reached the end of the data. LOGGER.info("Closed the bounded fake source"); From 6c443235475c2e7fb9040084cf1d06dc2bf0220d Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 13 Aug 2022 22:44:05 +0800 Subject: [PATCH 03/23] check style --- .../connector-fake/pom.xml | 1 + .../seatunnel/fake/source/FakeData.java | 19 +++++++++---------- .../seatunnel/fake/source/FakeSource.java | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/pom.xml b/seatunnel-connectors-v2/connector-fake/pom.xml index 60bb5042e36..7cb8ed3e4e2 100644 --- a/seatunnel-connectors-v2/connector-fake/pom.xml +++ b/seatunnel-connectors-v2/connector-fake/pom.xml @@ -28,6 +28,7 @@ 4.0.0 connector-fake + org.apache.seatunnel diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java index 0062314d267..a0b125534de 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -1,19 +1,18 @@ package org.apache.seatunnel.connectors.seatunnel.fake.source; -import java.math.BigDecimal; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; -/** - * - **/ +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; + +import java.math.BigDecimal; + public class FakeData { - public static final String[] columnName = new String[]{ + public static final String[] COLUMN_NAME = new String[]{ "c_void", "c_boolean", "c_byte", @@ -25,7 +24,7 @@ public class FakeData { "c_string", "c_decimal" }; - public static final SeaTunnelDataType[] columnType = new SeaTunnelDataType[]{ + public static final SeaTunnelDataType[] COLUMN_TYPE = new SeaTunnelDataType[]{ BasicType.VOID_TYPE, BasicType.BOOLEAN_TYPE, BasicType.BYTE_TYPE, @@ -38,7 +37,7 @@ public class FakeData { new DecimalType(38, 16) }; - + @SuppressWarnings("magicnumber") public static SeaTunnelRow generateRow() { Object[] columnValue = { Void.TYPE, @@ -52,7 +51,7 @@ public static SeaTunnelRow generateRow() { RandomStringUtils.random(10), BigDecimal.valueOf(RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE)) }; - if(columnValue.length != columnValue.length || columnValue.length != columnType.length){ + if (columnValue.length != columnValue.length || columnValue.length != COLUMN_TYPE.length) { throw new RuntimeException("the row data should be equals to column"); } return new SeaTunnelRow(columnValue); diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java index 18c63dd4f41..b97f852844e 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java @@ -45,8 +45,8 @@ public Boundedness getBoundedness() { @Override public SeaTunnelRowType getProducedType() { return new SeaTunnelRowType( - FakeData.columnName, - FakeData.columnType); + FakeData.COLUMN_NAME, + FakeData.COLUMN_TYPE); } @Override From 140e46ff8e983f29d06d010905cf52f08c3c3751 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sat, 13 Aug 2022 23:34:35 +0800 Subject: [PATCH 04/23] add license --- .../seatunnel/fake/source/FakeData.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java index a0b125534de..677bb818193 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.fake.source; import org.apache.seatunnel.api.table.type.BasicType; From 87842e426c01b2a8ac89976731050edb46564c28 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 14 Aug 2022 01:18:09 +0800 Subject: [PATCH 05/23] add data type --- .../seatunnel/fake/source/FakeData.java | 23 +++++++++++++++---- .../main/resources/examples/spark.batch.conf | 4 ++-- .../serialization/InternalRowConverter.java | 1 + 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java index 677bb818193..9bcc0ca5baf 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -17,8 +17,11 @@ package org.apache.seatunnel.connectors.seatunnel.fake.source; +import java.time.LocalDateTime; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; @@ -39,7 +42,11 @@ public class FakeData { "c_float", "c_double", "c_string", - "c_decimal" + "c_decimal", + "c_local_date_time", + "c_local_date", + "c_local_time", + "c_byte_arr" }; public static final SeaTunnelDataType[] COLUMN_TYPE = new SeaTunnelDataType[]{ BasicType.VOID_TYPE, @@ -51,7 +58,11 @@ public class FakeData { BasicType.FLOAT_TYPE, BasicType.DOUBLE_TYPE, BasicType.STRING_TYPE, - new DecimalType(38, 16) + new DecimalType(38, 18), + LocalTimeType.LOCAL_DATE_TIME_TYPE, + LocalTimeType.LOCAL_DATE_TYPE, + LocalTimeType.LOCAL_TIME_TYPE, + PrimitiveByteArrayType.INSTANCE }; @SuppressWarnings("magicnumber") @@ -65,8 +76,12 @@ public static SeaTunnelRow generateRow() { RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE), RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE), RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE), - RandomStringUtils.random(10), - BigDecimal.valueOf(RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE)) + RandomStringUtils.randomAlphabetic(10), + BigDecimal.valueOf(RandomUtils.nextFloat(0, Long.MAX_VALUE)), + LocalDateTime.now(), + LocalDateTime.now().toLocalDate(), + LocalDateTime.now().toLocalTime(), + RandomStringUtils.randomAlphabetic(10).getBytes(), }; if (columnValue.length != columnValue.length || columnValue.length != COLUMN_TYPE.length) { throw new RuntimeException("the row data should be equals to column"); diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index 37be7ca17b8..e007f07ef2d 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -34,7 +34,7 @@ source { # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { result_table_name = "fake" - field_name = "c_void, c_boolean, c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal" + field_name = "c_void, c_boolean, c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal, c_local_date_time, c_local_date, c_local_time, c_byte_arr" } # You can also use other input plugins, such as hdfs @@ -53,7 +53,7 @@ transform { # you can also use other transform plugins, such as sql sql { - sql = "select c_void, c_boolean,c_byte, c_short, c_int, c_long, c_float, c_double, c_string from fake" + sql = "select c_boolean,c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal, c_byte_arr from fake" result_table_name = "sql" } diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java index 0224654d6a8..5bbbd66a2dd 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.translation.spark.common.serialization; +import java.math.BigDecimal; import org.apache.seatunnel.api.table.type.MapType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; From 659ac9e37334b370eab8edbf802fac80cf9f6fd4 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 14 Aug 2022 09:34:21 +0800 Subject: [PATCH 06/23] update decimalType random --- .../seatunnel/connectors/seatunnel/fake/source/FakeData.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java index 9bcc0ca5baf..868a197eb29 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -77,7 +77,7 @@ public static SeaTunnelRow generateRow() { RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE), RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE), RandomStringUtils.randomAlphabetic(10), - BigDecimal.valueOf(RandomUtils.nextFloat(0, Long.MAX_VALUE)), + BigDecimal.valueOf(RandomUtils.nextLong(Integer.MAX_VALUE,Long.MAX_VALUE),18), LocalDateTime.now(), LocalDateTime.now().toLocalDate(), LocalDateTime.now().toLocalTime(), @@ -89,4 +89,7 @@ public static SeaTunnelRow generateRow() { return new SeaTunnelRow(columnValue); } + public static void main(String[] args) { + System.out.println(BigDecimal.valueOf(RandomUtils.nextDouble(0, Long.MAX_VALUE))); + } } From 7bad2a6d8e798f56c73aee44e2daedd4ed77b0c4 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 14 Aug 2022 09:37:52 +0800 Subject: [PATCH 07/23] update decimalType random --- .../connectors/seatunnel/fake/source/FakeData.java | 8 ++------ .../spark/common/serialization/InternalRowConverter.java | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java index 868a197eb29..1639e5b0e49 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.fake.source; -import java.time.LocalDateTime; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; @@ -29,6 +28,7 @@ import org.apache.commons.lang3.RandomUtils; import java.math.BigDecimal; +import java.time.LocalDateTime; public class FakeData { @@ -77,7 +77,7 @@ public static SeaTunnelRow generateRow() { RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE), RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE), RandomStringUtils.randomAlphabetic(10), - BigDecimal.valueOf(RandomUtils.nextLong(Integer.MAX_VALUE,Long.MAX_VALUE),18), + BigDecimal.valueOf(RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE), 18), LocalDateTime.now(), LocalDateTime.now().toLocalDate(), LocalDateTime.now().toLocalTime(), @@ -88,8 +88,4 @@ public static SeaTunnelRow generateRow() { } return new SeaTunnelRow(columnValue); } - - public static void main(String[] args) { - System.out.println(BigDecimal.valueOf(RandomUtils.nextDouble(0, Long.MAX_VALUE))); - } } diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java index 5bbbd66a2dd..decd29efd44 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.translation.spark.common.serialization; -import java.math.BigDecimal; import org.apache.seatunnel.api.table.type.MapType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; @@ -36,7 +35,6 @@ import org.apache.spark.sql.catalyst.expressions.MutableShort; import org.apache.spark.sql.catalyst.expressions.MutableValue; import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow; -import org.apache.spark.sql.types.Decimal; import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; @@ -83,8 +81,6 @@ private static Object convert(Object field, SeaTunnelDataType dataType) { return convertMap((Map) field, (MapType) dataType, InternalRowConverter::convert); case STRING: return UTF8String.fromString((String) field); - case DECIMAL: - return Decimal.apply((BigDecimal) field); default: return field; } From 5ed56bc5372d2f878e3770484c5fb2e62cfdc0ed Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Tue, 16 Aug 2022 00:47:27 +0800 Subject: [PATCH 08/23] support user-schema --- .../seatunnel/fake/source/FakeData.java | 91 --------------- .../seatunnel/fake/source/FakeRandomData.java | 104 ++++++++++++++++++ .../seatunnel/fake/source/FakeSource.java | 9 +- .../fake/source/FakeSourceReader.java | 7 +- 4 files changed, 114 insertions(+), 97 deletions(-) delete mode 100644 seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java create mode 100644 seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java deleted file mode 100644 index 1639e5b0e49..00000000000 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeData.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.fake.source; - -import org.apache.seatunnel.api.table.type.BasicType; -import org.apache.seatunnel.api.table.type.DecimalType; -import org.apache.seatunnel.api.table.type.LocalTimeType; -import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; -import org.apache.seatunnel.api.table.type.SeaTunnelDataType; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; - -import java.math.BigDecimal; -import java.time.LocalDateTime; - -public class FakeData { - - public static final String[] COLUMN_NAME = new String[]{ - "c_void", - "c_boolean", - "c_byte", - "c_short", - "c_int", - "c_long", - "c_float", - "c_double", - "c_string", - "c_decimal", - "c_local_date_time", - "c_local_date", - "c_local_time", - "c_byte_arr" - }; - public static final SeaTunnelDataType[] COLUMN_TYPE = new SeaTunnelDataType[]{ - BasicType.VOID_TYPE, - BasicType.BOOLEAN_TYPE, - BasicType.BYTE_TYPE, - BasicType.SHORT_TYPE, - BasicType.INT_TYPE, - BasicType.LONG_TYPE, - BasicType.FLOAT_TYPE, - BasicType.DOUBLE_TYPE, - BasicType.STRING_TYPE, - new DecimalType(38, 18), - LocalTimeType.LOCAL_DATE_TIME_TYPE, - LocalTimeType.LOCAL_DATE_TYPE, - LocalTimeType.LOCAL_TIME_TYPE, - PrimitiveByteArrayType.INSTANCE - }; - - @SuppressWarnings("magicnumber") - public static SeaTunnelRow generateRow() { - Object[] columnValue = { - Void.TYPE, - RandomUtils.nextInt(0, 2) == 1, - (byte) RandomUtils.nextInt(0, Byte.MAX_VALUE), - (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE), - RandomUtils.nextInt(Short.MAX_VALUE, Integer.MAX_VALUE), - RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE), - RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE), - RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE), - RandomStringUtils.randomAlphabetic(10), - BigDecimal.valueOf(RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE), 18), - LocalDateTime.now(), - LocalDateTime.now().toLocalDate(), - LocalDateTime.now().toLocalTime(), - RandomStringUtils.randomAlphabetic(10).getBytes(), - }; - if (columnValue.length != columnValue.length || columnValue.length != COLUMN_TYPE.length) { - throw new RuntimeException("the row data should be equals to column"); - } - return new SeaTunnelRow(columnValue); - } -} diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java new file mode 100644 index 00000000000..e081bde9fbe --- /dev/null +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.fake.source; + +import static org.apache.seatunnel.api.table.type.BasicType.BOOLEAN_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.BYTE_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.DOUBLE_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.FLOAT_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.INT_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.LONG_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.SHORT_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.STRING_TYPE; + +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeatunnelSchema; + +public class FakeRandomData { + + private final SeatunnelSchema schema; + + public FakeRandomData(SeatunnelSchema schema) { + this.schema = schema; + } + + public SeaTunnelRow randomRow() { + SeaTunnelRowType seaTunnelRowType = schema.getSeaTunnelRowType(); + String[] fieldNames = seaTunnelRowType.getFieldNames(); + SeaTunnelDataType[] fieldTypes = seaTunnelRowType.getFieldTypes(); + List randomRow = new ArrayList<>(fieldNames.length); + for (SeaTunnelDataType fieldType : fieldTypes) { + randomRow.add(randomColumnValue(fieldType)); + } + return new SeaTunnelRow(randomRow.toArray()); + } + + private Object randomColumnValue(SeaTunnelDataType fieldType) { + if (BOOLEAN_TYPE.equals(fieldType)) { + return RandomUtils.nextInt(0, 2) == 1; + } else if (BYTE_TYPE.equals(fieldType)) { + return (byte) RandomUtils.nextInt(0, Byte.MAX_VALUE); + } else if (SHORT_TYPE.equals(fieldType)) { + return (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE); + } else if (INT_TYPE.equals(fieldType)) { + return RandomUtils.nextInt(Short.MAX_VALUE, Integer.MAX_VALUE); + } else if (LONG_TYPE.equals(fieldType)) { + return RandomUtils.nextLong(Integer.MAX_VALUE, Long.MAX_VALUE); + } else if (FLOAT_TYPE.equals(fieldType)) { + return RandomUtils.nextFloat(Float.MIN_VALUE, Float.MAX_VALUE); + } else if (DOUBLE_TYPE.equals(fieldType)) { + return RandomUtils.nextDouble(Float.MAX_VALUE, Double.MAX_VALUE); + } else if (STRING_TYPE.equals(fieldType)) { + return RandomStringUtils.randomAlphabetic(10); + } else if (LocalTimeType.LOCAL_DATE_TYPE.equals(fieldType)) { + return randomLocalDateTime().toLocalDate(); + } else if (LocalTimeType.LOCAL_TIME_TYPE.equals(fieldType)) { + return randomLocalDateTime().toLocalTime(); + } else if (LocalTimeType.LOCAL_DATE_TIME_TYPE.equals(fieldType)) { + return randomLocalDateTime(); + } else if (fieldType instanceof DecimalType) { + DecimalType decimalType = (DecimalType) fieldType; + return new BigDecimal(RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()) + "." + + RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale())); + } else { + // todo: complex column + throw new IllegalStateException("Unexpected value: " + fieldType); + } + } + + + private LocalDateTime randomLocalDateTime() { + return LocalDateTime.of( + LocalDateTime.now().getYear(), + RandomUtils.nextInt(1, 12), + RandomUtils.nextInt(1, LocalDateTime.now().getDayOfMonth()), + RandomUtils.nextInt(0, 24), + RandomUtils.nextInt(0, 59) + ); + } +} diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java index b97f852844e..18ef25c1f71 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java @@ -23,6 +23,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeatunnelSchema; import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitReader; import org.apache.seatunnel.connectors.seatunnel.common.source.AbstractSingleSplitSource; import org.apache.seatunnel.connectors.seatunnel.common.source.SingleSplitReaderContext; @@ -36,6 +37,7 @@ public class FakeSource extends AbstractSingleSplitSource { private Config pluginConfig; private SeaTunnelContext seaTunnelContext; + private SeatunnelSchema schema; @Override public Boundedness getBoundedness() { @@ -44,14 +46,12 @@ public Boundedness getBoundedness() { @Override public SeaTunnelRowType getProducedType() { - return new SeaTunnelRowType( - FakeData.COLUMN_NAME, - FakeData.COLUMN_TYPE); + return schema.getSeaTunnelRowType(); } @Override public AbstractSingleSplitReader createReader(SingleSplitReaderContext readerContext) throws Exception { - return new FakeSourceReader(readerContext); + return new FakeSourceReader(readerContext, new FakeRandomData(schema)); } @Override @@ -62,6 +62,7 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) { this.pluginConfig = pluginConfig; + this.schema = SeatunnelSchema.buildWithConfig(pluginConfig); } @Override diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java index 5b413d81663..bd741f7d200 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java @@ -32,8 +32,11 @@ public class FakeSourceReader extends AbstractSingleSplitReader { private final SingleSplitReaderContext context; - public FakeSourceReader(SingleSplitReaderContext context) { + private final FakeRandomData fakeRandomData; + + public FakeSourceReader(SingleSplitReaderContext context,FakeRandomData randomData) { this.context = context; + this.fakeRandomData = randomData; } @Override @@ -51,7 +54,7 @@ public void close() { public void pollNext(Collector output) throws InterruptedException { // Generate a random number of rows to emit. for (int i = 0; i < 10; i++) { - SeaTunnelRow seaTunnelRow = FakeData.generateRow(); + SeaTunnelRow seaTunnelRow = fakeRandomData.randomRow(); output.collect(seaTunnelRow); } if (Boundedness.BOUNDED.equals(context.getBoundedness())) { From 1273c10f2149a98ebcc4e34b621b0a6640681530 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Wed, 17 Aug 2022 08:36:48 +0800 Subject: [PATCH 09/23] support user-schema --- .../common/schema/SeatunnelSchema.java | 4 ++- .../seatunnel/fake/source/FakeRandomData.java | 25 ++++++++++++++++++- .../fake/source/FakeSourceReader.java | 1 + .../main/resources/examples/spark.batch.conf | 21 ++++++++++++++-- .../serialization/InternalRowConverter.java | 1 + 5 files changed, 48 insertions(+), 4 deletions(-) diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java index 4c4799b21bd..a2dd3d87e26 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.common.schema; + import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -33,8 +34,9 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import java.util.Map; +import java.io.Serializable; -public class SeatunnelSchema { +public class SeatunnelSchema implements Serializable { public static final String SCHEMA = "schema"; private static final String FIELD_KEY = "fields"; private static final String SIMPLE_SCHEMA_FILED = "content"; diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java index e081bde9fbe..a9218b87f21 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -25,15 +25,21 @@ import static org.apache.seatunnel.api.table.type.BasicType.LONG_TYPE; import static org.apache.seatunnel.api.table.type.BasicType.SHORT_TYPE; import static org.apache.seatunnel.api.table.type.BasicType.STRING_TYPE; +import static org.apache.seatunnel.api.table.type.BasicType.VOID_TYPE; +import java.lang.reflect.Array; import java.math.BigDecimal; import java.time.LocalDateTime; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; @@ -85,8 +91,25 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { DecimalType decimalType = (DecimalType) fieldType; return new BigDecimal(RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()) + "." + RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale())); + } else if (fieldType instanceof ArrayType) { + ArrayType arrayType = (ArrayType) fieldType; + BasicType elementType = arrayType.getElementType(); + Object value = randomColumnValue(elementType); + Object arr = Array.newInstance(elementType.getTypeClass(), 1); + Array.set(arr, 0, value); + return arr; + } else if (fieldType instanceof MapType) { + MapType mapType = (MapType) fieldType; + SeaTunnelDataType keyType = mapType.getKeyType(); + Object key = randomColumnValue(keyType); + SeaTunnelDataType valueType = mapType.getValueType(); + Object value = randomColumnValue(valueType); + return new HashMap() {{ + put(key, value); + }}; + } else if (VOID_TYPE.equals(fieldType) || fieldType == null) { + return Void.TYPE; } else { - // todo: complex column throw new IllegalStateException("Unexpected value: " + fieldType); } } diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java index bd741f7d200..5b7b5827148 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java @@ -55,6 +55,7 @@ public void pollNext(Collector output) throws InterruptedException // Generate a random number of rows to emit. for (int i = 0; i < 10; i++) { SeaTunnelRow seaTunnelRow = fakeRandomData.randomRow(); + System.out.println(seaTunnelRow); output.collect(seaTunnelRow); } if (Boundedness.BOUNDED.equals(context.getBoundedness())) { diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index e007f07ef2d..bef43eefd73 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -33,8 +33,25 @@ env { source { # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "c_void, c_boolean, c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal, c_local_date_time, c_local_date, c_local_time, c_byte_arr" } # You can also use other input plugins, such as hdfs @@ -53,7 +70,7 @@ transform { # you can also use other transform plugins, such as sql sql { - sql = "select c_boolean,c_byte, c_short, c_int, c_long, c_float, c_double, c_string, c_decimal, c_byte_arr from fake" + sql = "select c_string,c_array from fake" result_table_name = "sql" } diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java index decd29efd44..6e728e78800 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java @@ -35,6 +35,7 @@ import org.apache.spark.sql.catalyst.expressions.MutableShort; import org.apache.spark.sql.catalyst.expressions.MutableValue; import org.apache.spark.sql.catalyst.expressions.SpecificInternalRow; +import org.apache.spark.sql.types.Decimal; import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; From 94f4e37da05eaebf36934857aa5111ec88ca2d7e Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Thu, 18 Aug 2022 23:00:54 +0800 Subject: [PATCH 10/23] update unit --- .../common/schema/SeatunnelSchema.java | 2 +- .../seatunnel/fake/source/FakeRandomData.java | 30 ++++--- .../fake/source/FakeSourceReader.java | 3 +- .../FakeRandomDataTest.java | 87 +++++++++++++++++++ .../src/test/resources/complex.schema.conf | 38 ++++++++ .../src/test/resources/simple.schema.conf | 37 ++++++++ .../main/resources/examples/spark.batch.conf | 2 +- .../serialization/InternalRowConverter.java | 1 - 8 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-fake/src/test/java/org.apache.seatunnel.connectors.seatunnel.fake.source/FakeRandomDataTest.java create mode 100644 seatunnel-connectors-v2/connector-fake/src/test/resources/complex.schema.conf create mode 100644 seatunnel-connectors-v2/connector-fake/src/test/resources/simple.schema.conf diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java index a2dd3d87e26..4d83c5b4dee 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java @@ -17,7 +17,7 @@ package org.apache.seatunnel.connectors.seatunnel.common.schema; - +import java.io.Serializable; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java index a9218b87f21..9a8aa1888a9 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -27,24 +27,27 @@ import static org.apache.seatunnel.api.table.type.BasicType.STRING_TYPE; import static org.apache.seatunnel.api.table.type.BasicType.VOID_TYPE; -import java.lang.reflect.Array; -import java.math.BigDecimal; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.RandomUtils; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; import org.apache.seatunnel.api.table.type.LocalTimeType; import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeatunnelSchema; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; + +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + public class FakeRandomData { private final SeatunnelSchema schema; @@ -64,6 +67,7 @@ public SeaTunnelRow randomRow() { return new SeaTunnelRow(randomRow.toArray()); } + @SuppressWarnings("magicnumber") private Object randomColumnValue(SeaTunnelDataType fieldType) { if (BOOLEAN_TYPE.equals(fieldType)) { return RandomUtils.nextInt(0, 2) == 1; @@ -104,9 +108,11 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { Object key = randomColumnValue(keyType); SeaTunnelDataType valueType = mapType.getValueType(); Object value = randomColumnValue(valueType); - return new HashMap() {{ - put(key, value); - }}; + HashMap objectObjectHashMap = new HashMap<>(); + objectObjectHashMap.put(key, value); + return objectObjectHashMap; + } else if (fieldType instanceof PrimitiveByteArrayType) { + return RandomUtils.nextBytes(10); } else if (VOID_TYPE.equals(fieldType) || fieldType == null) { return Void.TYPE; } else { @@ -114,7 +120,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { } } - + @SuppressWarnings("magicnumber") private LocalDateTime randomLocalDateTime() { return LocalDateTime.of( LocalDateTime.now().getYear(), diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java index 5b7b5827148..d67e1f4b215 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSourceReader.java @@ -34,7 +34,7 @@ public class FakeSourceReader extends AbstractSingleSplitReader { private final FakeRandomData fakeRandomData; - public FakeSourceReader(SingleSplitReaderContext context,FakeRandomData randomData) { + public FakeSourceReader(SingleSplitReaderContext context, FakeRandomData randomData) { this.context = context; this.fakeRandomData = randomData; } @@ -55,7 +55,6 @@ public void pollNext(Collector output) throws InterruptedException // Generate a random number of rows to emit. for (int i = 0; i < 10; i++) { SeaTunnelRow seaTunnelRow = fakeRandomData.randomRow(); - System.out.println(seaTunnelRow); output.collect(seaTunnelRow); } if (Boundedness.BOUNDED.equals(context.getBoundedness())) { diff --git a/seatunnel-connectors-v2/connector-fake/src/test/java/org.apache.seatunnel.connectors.seatunnel.fake.source/FakeRandomDataTest.java b/seatunnel-connectors-v2/connector-fake/src/test/java/org.apache.seatunnel.connectors.seatunnel.fake.source/FakeRandomDataTest.java new file mode 100644 index 00000000000..67906a1616d --- /dev/null +++ b/seatunnel-connectors-v2/connector-fake/src/test/java/org.apache.seatunnel.connectors.seatunnel.fake.source/FakeRandomDataTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.fake.source; + +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.api.table.type.SqlType; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeatunnelSchema; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.File; +import java.io.FileNotFoundException; +import java.lang.reflect.Array; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Paths; +import java.util.Map; + +public class FakeRandomDataTest { + + @ParameterizedTest + @ValueSource(strings = {"complex.schema.conf", "simple.schema.conf"}) + public void testComplexSchemaParse(String conf) throws FileNotFoundException, URISyntaxException { + Config testConfigFile = getTestConfigFile(conf); + SeatunnelSchema seatunnelSchema = SeatunnelSchema.buildWithConfig(testConfigFile); + FakeRandomData fakeRandomData = new FakeRandomData(seatunnelSchema); + SeaTunnelRow seaTunnelRow = fakeRandomData.randomRow(); + Assertions.assertNotNull(seaTunnelRow); + Object[] fields = seaTunnelRow.getFields(); + Assertions.assertNotNull(fields); + SeaTunnelRowType seaTunnelRowType = seatunnelSchema.getSeaTunnelRowType(); + SeaTunnelDataType[] fieldTypes = seaTunnelRowType.getFieldTypes(); + for (int i = 0; i < fieldTypes.length; i++) { + if (fieldTypes[i].getSqlType() != SqlType.NULL) { + Assertions.assertNotNull(fields[i]); + } else { + Assertions.assertSame(fields[i], Void.TYPE); + } + if (fieldTypes[i].getSqlType() == SqlType.MAP) { + Assertions.assertTrue(fields[i] instanceof Map); + Map field = (Map) fields[i]; + field.forEach((k, v) -> Assertions.assertTrue(k != null && v != null)); + } + if (fieldTypes[i].getSqlType() == SqlType.ARRAY) { + Assertions.assertTrue(fields[i].getClass().isArray()); + Assertions.assertNotNull(Array.get(fields[i], 0)); + } + } + } + + private Config getTestConfigFile(String configFile) throws FileNotFoundException, URISyntaxException { + if (!configFile.startsWith("/")) { + configFile = "/" + configFile; + } + URL resource = FakeRandomDataTest.class.getResource(configFile); + if (resource == null) { + throw new FileNotFoundException("Can't find config file: " + configFile); + } + String path = Paths.get(resource.toURI()).toString(); + Config config = ConfigFactory.parseFile(new File(path)); + assert config.hasPath("schema"); + return config.getConfig("schema"); + } + +} diff --git a/seatunnel-connectors-v2/connector-fake/src/test/resources/complex.schema.conf b/seatunnel-connectors-v2/connector-fake/src/test/resources/complex.schema.conf new file mode 100644 index 00000000000..6a06dbf06a6 --- /dev/null +++ b/seatunnel-connectors-v2/connector-fake/src/test/resources/complex.schema.conf @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +schema { + fields { + map = "map>" + map_array = "map>>" + array = "array" + string = string + boolean = boolean + tinyint = tinyint + smallint = smallint + int = int + bigint = bigint + float = float + double = double + decimal = "decimal(30, 8)" + null = "null" + bytes = bytes + date = date + time = time + timestamp = timestamp + } +} \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-fake/src/test/resources/simple.schema.conf b/seatunnel-connectors-v2/connector-fake/src/test/resources/simple.schema.conf new file mode 100644 index 00000000000..6716f00cdc2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-fake/src/test/resources/simple.schema.conf @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +schema { + fields { + map = "map" + array = "array" + string = string + boolean = boolean + tinyint = tinyint + smallint = smallint + int = int + bigint = bigint + float = float + double = double + decimal = "decimal(30, 8)" + null = "null" + bytes = bytes + date = date + time = time + timestamp = timestamp + } +} \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index bef43eefd73..0905e0d830f 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -70,7 +70,7 @@ transform { # you can also use other transform plugins, such as sql sql { - sql = "select c_string,c_array from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double,c_decimal,c_null,c_bytes from fake" result_table_name = "sql" } diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java index 6e728e78800..9de56707b36 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java @@ -39,7 +39,6 @@ import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; -import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; import java.time.LocalDate; From 93726f6e6f58713e3103e970aa246d74499f190f Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Thu, 18 Aug 2022 23:02:32 +0800 Subject: [PATCH 11/23] update local example --- .../src/main/resources/examples/spark.batch.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index 0905e0d830f..ddd664f3aa7 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -70,7 +70,7 @@ transform { # you can also use other transform plugins, such as sql sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double,c_decimal,c_null,c_bytes from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double,c_null,c_bytes from fake" result_table_name = "sql" } From f7f767ab8eacd024edd00e1477230f1b8691127c Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Thu, 18 Aug 2022 23:05:27 +0800 Subject: [PATCH 12/23] fix rebase --- .../spark/common/serialization/InternalRowConverter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java index 9de56707b36..0224654d6a8 100644 --- a/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java +++ b/seatunnel-translation/seatunnel-translation-spark/seatunnel-translation-spark-common/src/main/java/org/apache/seatunnel/translation/spark/common/serialization/InternalRowConverter.java @@ -39,6 +39,7 @@ import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; +import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; import java.time.LocalDate; @@ -81,6 +82,8 @@ private static Object convert(Object field, SeaTunnelDataType dataType) { return convertMap((Map) field, (MapType) dataType, InternalRowConverter::convert); case STRING: return UTF8String.fromString((String) field); + case DECIMAL: + return Decimal.apply((BigDecimal) field); default: return field; } From f5f9f20552a857672f9ee8211f4881f514b5de68 Mon Sep 17 00:00:00 2001 From: tangjiafu Date: Sun, 21 Aug 2022 12:04:13 +0800 Subject: [PATCH 13/23] update all test for fakeData --- .../seatunnel/fake/source/FakeRandomData.java | 6 +- .../assertion/fakesource_to_assert.conf | 104 +++++++++--------- .../resources/fake/fakesource_to_console.conf | 39 ++++--- .../file/fakesource_to_hdfs_json.conf | 53 +++++---- .../file/fakesource_to_hdfs_parquet.conf | 55 +++++---- .../file/fakesource_to_hdfs_text.conf | 55 +++++---- .../file/fakesource_to_local_json.conf | 53 +++++---- .../file/fakesource_to_local_parquet.conf | 54 +++++---- .../file/fakesource_to_local_text.conf | 55 +++++---- .../resources/iotdb/fakesource_to_iotdb.conf | 65 ++++++----- .../resources/jdbc/fakesource_to_jdbc.conf | 53 +++++---- .../resources/jdbc/jdbcsource_to_console.conf | 9 -- .../resources/fake/fakesource_to_console.conf | 55 ++++----- .../file/fakesource_to_hdfs_json.conf | 53 +++++---- .../file/fakesource_to_hdfs_parquet.conf | 55 +++++---- .../file/fakesource_to_hdfs_text.conf | 54 +++++---- .../file/fakesource_to_local_json.conf | 54 +++++---- .../file/fakesource_to_local_parquet.conf | 53 +++++---- .../file/fakesource_to_local_text.conf | 52 +++++---- .../resources/iotdb/fakesource_to_iotdb.conf | 68 +++++++----- .../resources/examples/fake_to_console.conf | 33 ++++-- .../resources/examples/fake_to_dingtalk.conf | 36 ++++-- .../examples/fakesource_to_file.conf | 36 ++++-- .../examples/spark.batch.clickhouse.conf | 31 +++++- .../main/resources/examples/spark.batch.conf | 6 +- 25 files changed, 721 insertions(+), 466 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java index 9a8aa1888a9..ff18abd9557 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -72,7 +72,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { if (BOOLEAN_TYPE.equals(fieldType)) { return RandomUtils.nextInt(0, 2) == 1; } else if (BYTE_TYPE.equals(fieldType)) { - return (byte) RandomUtils.nextInt(0, Byte.MAX_VALUE); + return (byte) RandomUtils.nextInt(Byte.MIN_VALUE, Byte.MAX_VALUE); } else if (SHORT_TYPE.equals(fieldType)) { return (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE); } else if (INT_TYPE.equals(fieldType)) { @@ -94,7 +94,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { } else if (fieldType instanceof DecimalType) { DecimalType decimalType = (DecimalType) fieldType; return new BigDecimal(RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()) + "." + - RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale())); + RandomStringUtils.randomNumeric(decimalType.getScale())); } else if (fieldType instanceof ArrayType) { ArrayType arrayType = (ArrayType) fieldType; BasicType elementType = arrayType.getElementType(); @@ -116,7 +116,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { } else if (VOID_TYPE.equals(fieldType) || fieldType == null) { return Void.TYPE; } else { - throw new IllegalStateException("Unexpected value: " + fieldType); + throw new UnsupportedOperationException("Unexpected value: " + fieldType); } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index d1e9c45839c..350970f9b60 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -25,64 +25,66 @@ env { #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } +# Docoment-Link +# https://seatunnel.apache.org/docs/category/source-v2 +# https://seatunnel.apache.org/docs/category/sink-v2 +# https://seatunnel.apache.org/docs/category/transform + source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name,age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + result_table_name = "fake" + } } -transform { - sql { - sql = "select name,age from fake" - } - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql +transform { + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } } sink { - Assert { - rules = - [{ - field_name = name - field_type = string - field_value = [ - { - rule_type = NOT_NULL - }, - { - rule_type = MIN_LENGTH - rule_value = 3 - }, - { - rule_type = MAX_LENGTH - rule_value = 20 - } - ] - },{ - field_name = age - field_type = int - field_value = [ - { - rule_type = NOT_NULL - }, - { - rule_type = MIN - rule_value = 1 - }, - { - rule_type = MAX - rule_value = 100 - } - ] - } - ] - } + Assert { + rules = [ + { + field_name = c_boolean + field_type = boolean + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_string + field_type = string + field_value = [ + { + rule_type = NOT_NULL + } + ] + } + ] + } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 49be0920fe8..18b09b10d3a 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -27,28 +27,37 @@ env { } source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name,age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + result_table_name = "fake" + } } transform { - sql { - sql = "select name,age from fake" - } + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { Console {} - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 769c8760de5..5174299653c 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -28,38 +28,51 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="json" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "json" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 9f5fd0b1787..2c5aecf98d9 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -28,39 +28,52 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="parquet" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "parquet" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index ef83dfd4e4a..accab04da4d 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -28,39 +28,52 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index b18b472f6cd..c0073edc01d 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -28,38 +28,51 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="json" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "json" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/LocalFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 9e2d5ad96af..f37728240f6 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -28,39 +28,51 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="parquet" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "parquet" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/LocalFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index d162b101f0c..bc28a5c540a 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -28,39 +28,52 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["age"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["name", "age"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/LocalFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index da1ae493678..5593e5f7c71 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -19,41 +19,54 @@ ###### env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { - FakeSource { - result_table_name = "fake" - field_name = "name, age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + result_table_name = "fake" + } } -transform { - sql { - sql = "select * from (values('root.ln.d1', '1660147200000', 'status,value', 'true,1001'), ('root.ln.d1', '1660233600000', 'status,value', 'false,1002')) t (device, `timestamp`, measurements, `values`)" - } - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql +transform { + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } } sink { - IoTDB { - node_urls = ["flink_e2e_iotdb_sink:6667"] - username = "root" - password = "root" - batch_size = 1 - batch_interval_ms = 10 - } + IoTDB { + node_urls = ["flink_e2e_iotdb_sink:6667"] + username = "root" + password = "root" + batch_size = 1 + batch_interval_ms = 10 + } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index 9640e19c228..5b4d959aec5 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -27,35 +27,46 @@ env { } source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + result_table_name = "fake" + } } -transform { - sql { - sql = "select name from fake" - } - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql +transform { + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } } - sink { Jdbc { - source_table_name = fake - driver = org.postgresql.Driver - url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" - user = test - password = test - query = "insert into test(name) values(?)" + source_table_name = fake + driver = org.postgresql.Driver + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" + user = test + password = test + query = "insert into test(name) values(?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Jdbc } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf index 6862abc04c2..3586dcacce0 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf @@ -27,7 +27,6 @@ env { } source { - # This is a example source plugin **only for test and demonstrate the feature source plugin** Jdbc { driver = org.postgresql.Driver url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" @@ -35,19 +34,11 @@ source { password = "test" query = "select * from test" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } transform { - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { Console {} - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 7b812762051..5762efc941a 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -30,44 +30,37 @@ env { } source { - # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { - result_table_name = "my_dataset" + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } + result_table_name = "fake" } - - # You can also use other input plugins, such as hdfs - # hdfs { - # result_table_name = "accesslog" - # path = "hdfs://hadoop-cluster-01/nginx/accesslog" - # format = "json" - # } - - # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake } -transform { - # split data by specific delimiter - - # you can also use other transform plugins, such as sql - # sql { - # sql = "select * from accesslog where request_time > 1000" - # } - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split +transform { + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } } sink { - # choose stdout output plugin to output data to console Console {} - - # you can also you other output plugins, such as sql - # hdfs { - # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed" - # save_mode = "append" - # } - - # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index c4d1aabe59b..32dc2917eca 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -27,38 +27,51 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="json" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "json" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index bdae80d74d1..0a0d4302e3a 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -27,39 +27,52 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="parquet" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "parquet" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index b682d3831cf..b04f98d4670 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -27,39 +27,51 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index d257f81bb44..4a10db0a550 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -27,38 +27,48 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="json" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "json" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } - - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index b5d1412120a..464c249f26c 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -27,37 +27,50 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } + transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="parquet" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "parquet" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 733a48e61a5..fdf54a60408 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -27,37 +27,49 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path="/tmp/hive/warehouse/test2" - field_delimiter="\t" - row_delimiter="\n" - partition_by=["age"] - partition_dir_expression="${k0}=${v0}" - is_partition_field_write_in_file=true - file_name_expression="${transactionId}_${now}" - file_format="text" - sink_columns=["name","age"] - filename_time_format="yyyy.MM.dd" - is_enable_transaction=true - save_mode="error" + path = "/tmp/hive/warehouse/test2" + field_delimiter = "\t" + row_delimiter = "\n" + partition_by = ["c_bigint"] + partition_dir_expression = "${k0}=${v0}" + is_partition_field_write_in_file = true + file_name_expression = "${transactionId}_${now}" + file_format = "text" + sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + filename_time_format = "yyyy.MM.dd" + is_enable_transaction = true + save_mode = "error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index 9c7e521b73d..c8e2228180a 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -16,43 +16,55 @@ # env { - # You can set spark configuration here - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local - job.mode = "BATCH" + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + job.mode = "BATCH" } source { - FakeSource { - result_table_name = "fake" - field_name = "name, age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - - # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource + result_table_name = "fake" + } } transform { - sql { - sql = "select * from (values('root.ln.d1', '1660147200000', 'status,value', 'true,1001'), ('root.ln.d1', '1660233600000', 'status,value', 'false,1002')) t (device, `timestamp`, measurements, `values`)" - } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" + } } sink { - IoTDB { - node_urls = ["spark_e2e_iotdb_sink:6668"] - username = "root" - password = "root" - batch_size = 1 - batch_interval_ms = 10 - } + IoTDB { + node_urls = ["spark_e2e_iotdb_sink:6668"] + username = "root" + password = "root" + batch_size = 1 + batch_interval_ms = 10 + } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf index feb394127ac..c9a8af0d8a9 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf @@ -28,27 +28,44 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name,age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - + result_table_name = "fake" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + result_table_name = "sql" } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { Console {} # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf index aed13429043..e7c2cc91aa6 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf @@ -28,22 +28,38 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name,age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - + result_table_name = "fake" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { - sql { - sql = "select name,age from fake" - } + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { @@ -53,5 +69,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf index f7b790c40b8..9a6589af1ca 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf @@ -28,22 +28,38 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - result_table_name = "fake" - field_name = "name,age" + FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp } - + result_table_name = "fake" + } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { + sql { + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + } - sql { - sql = "select name,age from fake" - } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { @@ -64,5 +80,5 @@ sink { } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf index 2e27410ee86..5c0654511b8 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf @@ -25,17 +25,42 @@ env { source { FakeSource { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } result_table_name = "fake" - field_name = "name,age" } + # If you would like to get more information about how to configure seatunnel and see full list of input plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { + # split data by specific delimiter + + # you can also use other transform plugins, such as sql sql { - sql = "select name,age from fake" + sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" result_table_name = "sql" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/category/transform } sink { @@ -43,7 +68,7 @@ sink { host = "139.198.158.103:8123" database = "default" table = "test_clickhouse_table_v2" - fields = ["name", "age"] + fields = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] username = 'default' bulk_size = 20000 retry_codes = [209, 210] diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index ddd664f3aa7..5784d9345a0 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -62,7 +62,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/2.1.3/category/source + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -75,7 +75,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/2.1.3/category/transform + # please go to https://seatunnel.apache.org/docs/category/transform } sink { @@ -89,5 +89,5 @@ sink { # } # If you would like to get more information about how to configure seatunnel and see full list of output plugins, - # please go to https://seatunnel.apache.org/docs/2.1.3/category/sink + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } From 564535948a93b8ac7136132991bd7472b5f1d400 Mon Sep 17 00:00:00 2001 From: tangjiafu Date: Sun, 21 Aug 2022 16:53:45 +0800 Subject: [PATCH 14/23] update all test for fakeData --- .../connectors/seatunnel/fake/source/FakeRandomData.java | 2 +- .../src/test/resources/assertion/fakesource_to_assert.conf | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java index ff18abd9557..c55f0219df4 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -72,7 +72,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { if (BOOLEAN_TYPE.equals(fieldType)) { return RandomUtils.nextInt(0, 2) == 1; } else if (BYTE_TYPE.equals(fieldType)) { - return (byte) RandomUtils.nextInt(Byte.MIN_VALUE, Byte.MAX_VALUE); + return (byte) RandomUtils.nextInt(0, 255); } else if (SHORT_TYPE.equals(fieldType)) { return (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE); } else if (INT_TYPE.equals(fieldType)) { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index 350970f9b60..ebca1f7c45b 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -25,11 +25,6 @@ env { #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } -# Docoment-Link -# https://seatunnel.apache.org/docs/category/source-v2 -# https://seatunnel.apache.org/docs/category/sink-v2 -# https://seatunnel.apache.org/docs/category/transform - source { FakeSource { fields { From 2b30f1e577a28d5a0cce03cee6c7c1a2055740d1 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 21:33:05 +0800 Subject: [PATCH 15/23] update flink e2e conf --- .../assertion/fakesource_to_assert.conf | 95 ++++++++++--------- .../resources/fake/fakesource_to_console.conf | 24 +---- .../file/fakesource_to_hdfs_json.conf | 48 ++++------ .../file/fakesource_to_hdfs_parquet.conf | 47 ++++----- .../file/fakesource_to_hdfs_text.conf | 48 ++++------ .../file/fakesource_to_local_json.conf | 45 +++------ .../file/fakesource_to_local_parquet.conf | 47 ++++----- .../file/fakesource_to_local_text.conf | 47 ++++----- .../resources/iotdb/fakesource_to_iotdb.conf | 64 +++++-------- .../resources/jdbc/fakesource_to_jdbc.conf | 50 ++++------ .../resources/jdbc/jdbcsource_to_console.conf | 9 ++ 11 files changed, 206 insertions(+), 318 deletions(-) diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index ebca1f7c45b..30b5aaa71ae 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -26,60 +26,63 @@ env { } source { - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp - } - result_table_name = "fake" + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" - } + sql { + sql = "select name,age from fake" + } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { - Assert { - rules = [ - { - field_name = c_boolean - field_type = boolean - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - { - field_name = c_string - field_type = string - field_value = [ - { - rule_type = NOT_NULL - } - ] - } - ] - } + Assert { + rules = + [{ + field_name = name + field_type = string + field_value = [ + { + rule_type = NOT_NULL + }, + { + rule_type = MIN_LENGTH + rule_value = 3 + }, + { + rule_type = MAX_LENGTH + rule_value = 20 + } + ] + },{ + field_name = age + field_type = int + field_value = [ + { + rule_type = NOT_NULL + }, + { + rule_type = MIN + rule_value = 1 + }, + { + rule_type = MAX + rule_value = 100 + } + ] + } + ] + } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, # please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 18b09b10d3a..3f9d904a176 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -27,33 +27,19 @@ env { } source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 5174299653c..ededd75a09f 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -28,49 +28,37 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "json" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="json" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 2c5aecf98d9..07694a243bf 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -28,50 +28,35 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "parquet" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="parquet" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index accab04da4d..1b318f452ea 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -28,50 +28,34 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } - transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "text" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index c0073edc01d..7106092dfa5 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -28,49 +28,34 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "json" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="json" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index f37728240f6..1a03dbffecb 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -28,49 +28,34 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "parquet" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="parquet" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index bc28a5c540a..3fca6ed2473 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -28,50 +28,35 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["age"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "text" - sink_columns = ["name", "age"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index 5593e5f7c71..eece9f8ebca 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -19,54 +19,42 @@ ###### env { - # You can set flink configuration here - execution.parallelism = 1 - job.mode = "BATCH" - #execution.checkpoint.interval = 10000 - #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" + # You can set flink configuration here + execution.parallelism = 1 + job.mode = "BATCH" + #execution.checkpoint.interval = 10000 + #execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint" } source { - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" - } + sql { + sql = "select * from (values('root.ln.d1', '1660147200000', 'status,value', 'true,1001'), ('root.ln.d1', '1660233600000', 'status,value', 'false,1002')) t (device, `timestamp`, measurements, `values`)" + } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { - IoTDB { - node_urls = ["flink_e2e_iotdb_sink:6667"] - username = "root" - password = "root" - batch_size = 1 - batch_interval_ms = 10 - } + IoTDB { + node_urls = ["flink_e2e_iotdb_sink:6667"] + username = "root" + password = "root" + batch_size = 1 + batch_interval_ms = 10 + } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index 5b4d959aec5..726c05427f2 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -27,44 +27,34 @@ env { } source { - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + # This is a example source plugin **only for test and demonstrate the feature source plugin** + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" - } + sql { + sql = "select name from fake" + } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } + sink { Jdbc { - source_table_name = fake - driver = org.postgresql.Driver - url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" - user = test - password = test - query = "insert into test(name) values(?)" + source_table_name = fake + driver = org.postgresql.Driver + url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" + user = test + password = test + query = "insert into test(name) values(?)" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf index 3586dcacce0..6862abc04c2 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf @@ -27,6 +27,7 @@ env { } source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** Jdbc { driver = org.postgresql.Driver url = "jdbc:postgresql://postgresql:5432/test?loggerLevel=OFF" @@ -34,11 +35,19 @@ source { password = "test" query = "select * from test" } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } transform { + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { Console {} + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console } \ No newline at end of file From 23fc0660d08794c52d47909944d6c15db934ce53 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 21:44:02 +0800 Subject: [PATCH 16/23] update e2e conf --- .../resources/fake/fakesource_to_console.conf | 55 ++++++++------ .../file/fakesource_to_hdfs_json.conf | 49 +++++-------- .../file/fakesource_to_hdfs_parquet.conf | 50 +++++-------- .../file/fakesource_to_hdfs_text.conf | 53 ++++++-------- .../file/fakesource_to_local_json.conf | 52 ++++++-------- .../file/fakesource_to_local_parquet.conf | 54 ++++++-------- .../file/fakesource_to_local_text.conf | 53 ++++++-------- .../resources/iotdb/fakesource_to_iotdb.conf | 71 ++++++++----------- .../resources/examples/fake_to_console.conf | 30 +++----- .../resources/examples/fake_to_dingtalk.conf | 33 +++------ .../examples/fakesource_to_file.conf | 33 +++------ .../examples/spark.batch.clickhouse.conf | 30 ++------ 12 files changed, 221 insertions(+), 342 deletions(-) diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 5762efc941a..57be60b61e9 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -30,37 +30,48 @@ env { } source { + # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { + result_table_name = "my_dataset" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } -} + # You can also use other input plugins, such as hdfs + # hdfs { + # result_table_name = "accesslog" + # path = "hdfs://hadoop-cluster-01/nginx/accesslog" + # format = "json" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of input plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake +} transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" - } + # split data by specific delimiter + + # you can also use other transform plugins, such as sql + # sql { + # sql = "select * from accesslog where request_time > 1000" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/transform-plugins/Split } sink { + # choose stdout output plugin to output data to console Console {} + + # you can also you other output plugins, such as sql + # hdfs { + # path = "hdfs://hadoop-cluster-01/nginx/accesslog_processed" + # save_mode = "append" + # } + + # If you would like to get more information about how to configure seatunnel and see full list of output plugins, + # please go to https://seatunnel.apache.org/docs/spark/configuration/sink-plugins/Console } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 32dc2917eca..12a06437b5c 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -27,49 +27,36 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } - transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "json" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="json" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 0a0d4302e3a..136c23265b8 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -27,50 +27,38 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "parquet" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="parquet" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index b04f98d4670..7efa0b90a23 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -27,49 +27,40 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { HdfsFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "text" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index 4a10db0a550..1d5b560876b 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -27,48 +27,38 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } } - transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "json" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="json" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/File } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 464c249f26c..7fe12f746b7 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -27,50 +27,40 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake +} transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "parquet" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="parquet" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index fdf54a60408..125fc194074 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -27,49 +27,40 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { LocalFile { - path = "/tmp/hive/warehouse/test2" - field_delimiter = "\t" - row_delimiter = "\n" - partition_by = ["c_bigint"] - partition_dir_expression = "${k0}=${v0}" - is_partition_field_write_in_file = true - file_name_expression = "${transactionId}_${now}" - file_format = "text" - sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] - filename_time_format = "yyyy.MM.dd" - is_enable_transaction = true - save_mode = "error" + path="/tmp/hive/warehouse/test2" + field_delimiter="\t" + row_delimiter="\n" + partition_by=["age"] + partition_dir_expression="${k0}=${v0}" + is_partition_field_write_in_file=true + file_name_expression="${transactionId}_${now}" + file_format="text" + sink_columns=["name","age"] + filename_time_format="yyyy.MM.dd" + is_enable_transaction=true + save_mode="error" } # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index c8e2228180a..3f887656d0b 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -16,55 +16,46 @@ # env { - # You can set spark configuration here - spark.app.name = "SeaTunnel" - spark.executor.instances = 2 - spark.executor.cores = 1 - spark.executor.memory = "1g" - spark.master = local - job.mode = "BATCH" + # You can set spark configuration here + spark.app.name = "SeaTunnel" + spark.executor.instances = 2 + spark.executor.cores = 1 + spark.executor.memory = "1g" + spark.master = local + job.mode = "BATCH" } source { - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" - } + sql { + sql = "select * from (values('root.ln.d1', '1660147200000', 'status,value', 'true,1001'), ('root.ln.d1', '1660233600000', 'status,value', 'false,1002')) t (device, `timestamp`, measurements, `values`)" + } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { - IoTDB { - node_urls = ["spark_e2e_iotdb_sink:6668"] - username = "root" - password = "root" - batch_size = 1 - batch_interval_ms = 10 - } + IoTDB { + node_urls = ["spark_e2e_iotdb_sink:6668"] + username = "root" + password = "root" + batch_size = 1 + batch_interval_ms = 10 + } - # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/sink/IoTDB } \ No newline at end of file diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf index c9a8af0d8a9..395008a35fa 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf @@ -28,35 +28,21 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - result_table_name = "sql" + sql = "select name,age from fake" } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf index e7c2cc91aa6..ed9fe2fa94c 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf @@ -28,35 +28,22 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - } + sql { + sql = "select name,age from fake" + } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, # please go to https://seatunnel.apache.org/docs/category/transform diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf index 9a6589af1ca..79c032a2486 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf @@ -28,36 +28,23 @@ env { source { # This is a example source plugin **only for test and demonstrate the feature source plugin** - FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + FakeSource { + result_table_name = "fake" + fields { + name = "string" + age = "int" + } } - result_table_name = "fake" - } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { - sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" - } + sql { + sql = "select name,age from fake" + } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, # please go to https://seatunnel.apache.org/docs/category/transform } diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf index 5c0654511b8..1d59892d791 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf @@ -25,25 +25,11 @@ env { source { FakeSource { + result_table_name = "fake" fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + name = "string" + age = "int" } - result_table_name = "fake" } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, # please go to https://seatunnel.apache.org/docs/category/source-v2 @@ -51,16 +37,10 @@ source { transform { - # split data by specific delimiter - - # you can also use other transform plugins, such as sql sql { - sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake" + sql = "select name,age from fake" result_table_name = "sql" } - - # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/category/transform } sink { @@ -68,7 +48,7 @@ sink { host = "139.198.158.103:8123" database = "default" table = "test_clickhouse_table_v2" - fields = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"] + fields = ["name", "age"] username = 'default' bulk_size = 20000 retry_codes = [209, 210] From f0d226d485c0a4dec5b4fbed3064bef6a2d1b8f0 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 21:50:26 +0800 Subject: [PATCH 17/23] update import --- .../connectors/seatunnel/common/schema/SeatunnelSchema.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java index 4d83c5b4dee..835d1d0ca58 100644 --- a/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java +++ b/seatunnel-connectors-v2/connector-common/src/main/java/org/apache/seatunnel/connectors/seatunnel/common/schema/SeatunnelSchema.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.common.schema; -import java.io.Serializable; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -33,8 +32,8 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; -import java.util.Map; import java.io.Serializable; +import java.util.Map; public class SeatunnelSchema implements Serializable { public static final String SCHEMA = "schema"; From ecdbae200cb66543e958997e9264878039a07309 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 22:06:53 +0800 Subject: [PATCH 18/23] fix dead link --- .../resources/assertion/fakesource_to_assert.conf | 1 + .../test/resources/fake/fakesource_to_console.conf | 13 ++++++++++--- .../resources/file/fakesource_to_hdfs_json.conf | 4 +++- .../resources/file/fakesource_to_hdfs_parquet.conf | 7 ++++++- .../resources/file/fakesource_to_hdfs_text.conf | 6 ++++++ .../resources/file/fakesource_to_local_json.conf | 7 ++++++- .../resources/file/fakesource_to_local_parquet.conf | 6 ++++++ .../resources/file/fakesource_to_local_text.conf | 7 ++++++- .../src/test/resources/jdbc/fakesource_to_jdbc.conf | 6 ++++-- .../test/resources/jdbc/jdbcsource_to_console.conf | 4 ++-- .../test/resources/fake/fakesource_to_console.conf | 2 +- .../resources/file/fakesource_to_hdfs_json.conf | 5 ++++- .../resources/file/fakesource_to_hdfs_parquet.conf | 6 ++++-- .../resources/file/fakesource_to_hdfs_text.conf | 4 ++-- .../resources/file/fakesource_to_local_json.conf | 5 ++++- .../resources/file/fakesource_to_local_parquet.conf | 4 ++-- .../resources/file/fakesource_to_local_text.conf | 4 ++-- 17 files changed, 69 insertions(+), 22 deletions(-) diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index 30b5aaa71ae..a9a1263a0ed 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -26,6 +26,7 @@ env { } source { + # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" fields { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 3f9d904a176..762dfdc5f20 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -35,15 +35,22 @@ source { age = "int" } } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { - sql { - sql = "select name,age from fake" - } + sql { + sql = "select name,age from fake" + } + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { Console {} + + # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index ededd75a09f..655b9c24c1c 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -34,8 +34,10 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 07694a243bf..9a450ef1cd1 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -34,13 +34,18 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index 1b318f452ea..afe31a329a8 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -34,12 +34,18 @@ source { age = "int" } } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { sql { sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index 7106092dfa5..1293fe74a2d 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -34,13 +34,18 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 1a03dbffecb..ccf1eaeb7f7 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -34,12 +34,18 @@ source { age = "int" } } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { sql { sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 3fca6ed2473..339e33b5faa 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -34,13 +34,18 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { sql = "select name,age from fake" } + + # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index 726c05427f2..a622e4c74f5 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -35,8 +35,10 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { @@ -44,7 +46,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf index 6862abc04c2..4d65c569ead 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/jdbcsource_to_console.conf @@ -43,11 +43,11 @@ source { transform { # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/transform/sql } sink { Console {} # If you would like to get more information about how to configure seatunnel and see full list of sink plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console + # please go to https://seatunnel.apache.org/docs/category/sink-v2 } \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 57be60b61e9..986cc151db5 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -47,7 +47,7 @@ source { # } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/spark/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 12a06437b5c..c2bb65442e4 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -33,6 +33,9 @@ source { age = "int" } } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -41,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 136c23265b8..931dbeaac16 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -33,8 +33,10 @@ source { age = "int" } } -} + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, +# please go to https://seatunnel.apache.org/docs/category/source-v2 +} transform { sql { @@ -42,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index 7efa0b90a23..85bb33f603f 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -35,7 +35,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -44,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index 1d5b560876b..f3ae5e061b8 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -33,6 +33,9 @@ source { age = "int" } } + + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -41,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 7fe12f746b7..136b61f4383 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -35,7 +35,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -44,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/transform/sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 125fc194074..871b810c0e0 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -35,7 +35,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake + # please go to https://seatunnel.apache.org/docs/category/source-v2 } transform { @@ -44,7 +44,7 @@ transform { } # If you would like to get more information about how to configure seatunnel and see full list of transform plugins, - # please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql + # please go to https://seatunnel.apache.org/docs/category/transform } sink { From 1f55a0fb19d3d39ef2b085d730da40c9fc42c1fb Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 22:42:00 +0800 Subject: [PATCH 19/23] update doc and schema define --- docs/en/connector-v2/source/FakeSource.md | 76 +++++++++++++++++++ .../seatunnel/fake/source/FakeRandomData.java | 4 +- .../seatunnel/fake/source/FakeSource.java | 3 +- .../assertion/fakesource_to_assert.conf | 8 +- .../resources/fake/fakesource_to_console.conf | 8 +- .../file/fakesource_to_hdfs_json.conf | 8 +- .../file/fakesource_to_hdfs_parquet.conf | 8 +- .../file/fakesource_to_hdfs_text.conf | 8 +- .../file/fakesource_to_local_json.conf | 8 +- .../file/fakesource_to_local_parquet.conf | 8 +- .../file/fakesource_to_local_text.conf | 8 +- .../resources/iotdb/fakesource_to_iotdb.conf | 8 +- .../resources/jdbc/fakesource_to_jdbc.conf | 8 +- .../resources/fake/fakesource_to_console.conf | 8 +- .../file/fakesource_to_hdfs_json.conf | 8 +- .../file/fakesource_to_hdfs_parquet.conf | 8 +- .../file/fakesource_to_hdfs_text.conf | 8 +- .../file/fakesource_to_local_json.conf | 8 +- .../file/fakesource_to_local_parquet.conf | 8 +- .../file/fakesource_to_local_text.conf | 8 +- .../resources/iotdb/fakesource_to_iotdb.conf | 10 ++- .../resources/examples/fake_to_console.conf | 8 +- .../resources/examples/fake_to_dingtalk.conf | 8 +- .../examples/fakesource_to_file.conf | 8 +- .../examples/spark.batch.clickhouse.conf | 8 +- .../main/resources/examples/spark.batch.conf | 36 ++++----- 26 files changed, 210 insertions(+), 87 deletions(-) create mode 100644 docs/en/connector-v2/source/FakeSource.md diff --git a/docs/en/connector-v2/source/FakeSource.md b/docs/en/connector-v2/source/FakeSource.md new file mode 100644 index 00000000000..913ab4368a3 --- /dev/null +++ b/docs/en/connector-v2/source/FakeSource.md @@ -0,0 +1,76 @@ +# LocalFile + +> FakeSource connector + +## Description + +The FakeSource is a virtual data source, which randomly generates the number of rows according to the data structure of the user-defined schema, +just for testing, such as type conversion and feature testing + +## Options + +| name | type | required | default value | +|-------------------|--------|----------|---------------| +| result_table_name | string | yes | - | +| schema | config | yes | - | + +### result_table_name [string] + +The table name. + +### type [string] +Table structure description ,you should assign schema option to tell connector how to parse data to the row you want. +**Tips**: Most of Unstructured-Datasource contain this param, such as LocalFile,HdfsFile. +**Example**: +```hocon +schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } + } +``` + + + +```hocon +source { + FakeSource { + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } + } + result_table_name = "fake" + } +} +``` \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java index c55f0219df4..efafde5caff 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java @@ -49,7 +49,7 @@ import java.util.List; public class FakeRandomData { - + public static final String SCHEMA = "schema"; private final SeatunnelSchema schema; public FakeRandomData(SeatunnelSchema schema) { @@ -112,7 +112,7 @@ private Object randomColumnValue(SeaTunnelDataType fieldType) { objectObjectHashMap.put(key, value); return objectObjectHashMap; } else if (fieldType instanceof PrimitiveByteArrayType) { - return RandomUtils.nextBytes(10); + return RandomUtils.nextBytes(100); } else if (VOID_TYPE.equals(fieldType) || fieldType == null) { return Void.TYPE; } else { diff --git a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java index 18ef25c1f71..4e7b1317e20 100644 --- a/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java +++ b/seatunnel-connectors-v2/connector-fake/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeSource.java @@ -62,7 +62,8 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) { this.pluginConfig = pluginConfig; - this.schema = SeatunnelSchema.buildWithConfig(pluginConfig); + assert pluginConfig.hasPath(FakeRandomData.SCHEMA); + this.schema = SeatunnelSchema.buildWithConfig(pluginConfig.getConfig(FakeRandomData.SCHEMA)); } @Override diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index a9a1263a0ed..40024d03b29 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -29,9 +29,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 762dfdc5f20..a2d85e44be0 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -30,9 +30,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 655b9c24c1c..42ef83f690d 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 9a450ef1cd1..bf9a5c2b033 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index afe31a329a8..9efb64184dd 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index 1293fe74a2d..f516d4f872f 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index ccf1eaeb7f7..32d83e88c34 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 339e33b5faa..d9fd7a07b99 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index eece9f8ebca..5fa44ccc8d5 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -29,9 +29,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index a622e4c74f5..f29426e5349 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -30,9 +30,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 986cc151db5..5674454adea 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -33,9 +33,11 @@ source { # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { result_table_name = "my_dataset" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index c2bb65442e4..ddf92ee0b2d 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 931dbeaac16..1d9a5b42700 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index 85bb33f603f..cf13de77d8e 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index f3ae5e061b8..c04a3d68945 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 136b61f4383..4c0474a94aa 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 871b810c0e0..9e9a813e621 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -28,9 +28,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index 3f887656d0b..503668487e4 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -28,10 +28,12 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" - } + schema = { + fields { + name = "string" + age = "int" + } + } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf index 395008a35fa..6a89b64a656 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_console.conf @@ -30,9 +30,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf index ed9fe2fa94c..d681985d030 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fake_to_dingtalk.conf @@ -30,9 +30,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf index 79c032a2486..d640281ccf8 100644 --- a/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf +++ b/seatunnel-examples/seatunnel-flink-connector-v2-example/src/main/resources/examples/fakesource_to_file.conf @@ -30,9 +30,11 @@ source { # This is a example source plugin **only for test and demonstrate the feature source plugin** FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf index 1d59892d791..8d57e1f3b92 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.clickhouse.conf @@ -26,9 +26,11 @@ env { source { FakeSource { result_table_name = "fake" - fields { - name = "string" - age = "int" + schema = { + fields { + name = "string" + age = "int" + } } } # If you would like to get more information about how to configure seatunnel and see full list of input plugins, diff --git a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf index 5784d9345a0..6e7755d8c53 100644 --- a/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf +++ b/seatunnel-examples/seatunnel-spark-connector-v2-example/src/main/resources/examples/spark.batch.conf @@ -33,23 +33,25 @@ env { source { # This is a example input plugin **only for test and demonstrate the feature input plugin** FakeSource { - fields { - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(30, 8)" - c_null = "null" - c_bytes = bytes - c_date = date - c_time = time - c_timestamp = timestamp + schema = { + fields { + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(30, 8)" + c_null = "null" + c_bytes = bytes + c_date = date + c_time = time + c_timestamp = timestamp + } } result_table_name = "fake" } From 7e3c4c4b3e7ae223b34efc80212a4860373c19fe Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 22:49:53 +0800 Subject: [PATCH 20/23] update fakeData link --- .../src/test/resources/assertion/fakesource_to_assert.conf | 3 ++- .../src/test/resources/fake/fakesource_to_console.conf | 2 +- .../src/test/resources/file/fakesource_to_hdfs_json.conf | 2 +- .../src/test/resources/file/fakesource_to_hdfs_parquet.conf | 2 +- .../src/test/resources/file/fakesource_to_hdfs_text.conf | 2 +- .../src/test/resources/file/fakesource_to_local_json.conf | 2 +- .../src/test/resources/file/fakesource_to_local_parquet.conf | 2 +- .../src/test/resources/file/fakesource_to_local_text.conf | 2 +- .../src/test/resources/iotdb/fakesource_to_iotdb.conf | 2 ++ .../src/test/resources/jdbc/fakesource_to_jdbc.conf | 2 +- .../src/test/resources/fake/fakesource_to_console.conf | 4 ++-- .../src/test/resources/file/fakesource_to_hdfs_json.conf | 2 +- .../src/test/resources/file/fakesource_to_hdfs_parquet.conf | 2 +- .../src/test/resources/file/fakesource_to_hdfs_text.conf | 2 +- .../src/test/resources/file/fakesource_to_local_json.conf | 2 +- .../src/test/resources/file/fakesource_to_local_parquet.conf | 2 +- .../src/test/resources/file/fakesource_to_local_text.conf | 2 +- 17 files changed, 20 insertions(+), 17 deletions(-) diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index 40024d03b29..1e1a6ac1c61 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -36,9 +36,10 @@ source { } } } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake } - transform { sql { sql = "select name,age from fake" diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index a2d85e44be0..d9eb0ff8614 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -38,7 +38,7 @@ source { } } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index 42ef83f690d..269b85d0840 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index bf9a5c2b033..5e1ea5c01d1 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index 9efb64184dd..d4a8a745cc2 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index f516d4f872f..c118c33c89a 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 32d83e88c34..c02c1614579 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index d9fd7a07b99..7d1a5d42ce0 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -38,7 +38,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf index 5fa44ccc8d5..4c5e0fe4e96 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/iotdb/fakesource_to_iotdb.conf @@ -36,6 +36,8 @@ source { } } } + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index f29426e5349..f7e975d9d4d 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -39,7 +39,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf index 5674454adea..f469338a517 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf @@ -48,8 +48,8 @@ source { # format = "json" # } - # If you would like to get more information about how to configure seatunnel and see full list of input plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # If you would like to get more information about how to configure seatunnel and see full list of source plugins, + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf index ddf92ee0b2d..40454bce061 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf index 1d9a5b42700..550990eeace 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, -# please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf index cf13de77d8e..2bf6afba61f 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_text.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf index c04a3d68945..f8dab0cbca1 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_json.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf index 4c0474a94aa..edbad7ec6f3 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_parquet.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { diff --git a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf index 9e9a813e621..41ff5e8f60b 100644 --- a/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf +++ b/seatunnel-e2e/seatunnel-spark-connector-v2-e2e/src/test/resources/file/fakesource_to_local_text.conf @@ -37,7 +37,7 @@ source { } # If you would like to get more information about how to configure seatunnel and see full list of source plugins, - # please go to https://seatunnel.apache.org/docs/category/source-v2 + # please go to https://seatunnel.apache.org/docs/connector-v2/source/FakeSource } transform { From 2194f225c5f38c11e9bd4d615571b699cdbc9728 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Sun, 21 Aug 2022 22:52:39 +0800 Subject: [PATCH 21/23] update doc --- docs/en/connector-v2/source/FakeSource.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/connector-v2/source/FakeSource.md b/docs/en/connector-v2/source/FakeSource.md index 913ab4368a3..09cd4a06e98 100644 --- a/docs/en/connector-v2/source/FakeSource.md +++ b/docs/en/connector-v2/source/FakeSource.md @@ -45,8 +45,8 @@ schema = { } ``` - - +## Example +Simple source for FakeSource which contains enough datatype ```hocon source { FakeSource { From 27bd47ec208821192ed9ef1274b13a52abd661f9 Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 22 Aug 2022 22:13:21 +0800 Subject: [PATCH 22/23] update assert --- docs/en/connector-v2/sink/Assert.md | 12 +++++------- .../resources/assertion/fakesource_to_assert.conf | 8 ++++---- .../src/test/resources/jdbc/fakesource_to_jdbc.conf | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/en/connector-v2/sink/Assert.md b/docs/en/connector-v2/sink/Assert.md index ca9332d0c58..9e5c49acfa4 100644 --- a/docs/en/connector-v2/sink/Assert.md +++ b/docs/en/connector-v2/sink/Assert.md @@ -37,13 +37,11 @@ A list value rule define the data value validation ### rule_type [string] The following rules are supported for now -` -NOT_NULL, // value can't be null -MIN, // define the minimum value of data -MAX, // define the maximum value of data -MIN_LENGTH, // define the minimum string length of a string data -MAX_LENGTH // define the maximum string length of a string data -` +- NOT_NULL `value can't be null` +- MIN `define the minimum value of data` +- MAX `define the maximum value of data` +- MIN_LENGTH `define the minimum string length of a string data` +- MAX_LENGTH `define the maximum string length of a string data` ### rule_value [double] diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf index 1e1a6ac1c61..b2fda476b3e 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf @@ -61,11 +61,11 @@ sink { }, { rule_type = MIN_LENGTH - rule_value = 3 + rule_value = 10 }, { rule_type = MAX_LENGTH - rule_value = 20 + rule_value = 10 } ] },{ @@ -77,11 +77,11 @@ sink { }, { rule_type = MIN - rule_value = 1 + rule_value = 32767 }, { rule_type = MAX - rule_value = 100 + rule_value = 2147483647 } ] } diff --git a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf index f7e975d9d4d..0f732b63b94 100644 --- a/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf +++ b/seatunnel-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/jdbc/fakesource_to_jdbc.conf @@ -33,7 +33,6 @@ source { schema = { fields { name = "string" - age = "int" } } } From 398081a20b3a00f5aa9af202a5d5d4e419a34f8a Mon Sep 17 00:00:00 2001 From: laglangyue <373435126@qq.com> Date: Mon, 22 Aug 2022 22:16:00 +0800 Subject: [PATCH 23/23] update doc title --- docs/en/connector-v2/source/FakeSource.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/connector-v2/source/FakeSource.md b/docs/en/connector-v2/source/FakeSource.md index 09cd4a06e98..9c4bf4ffd0a 100644 --- a/docs/en/connector-v2/source/FakeSource.md +++ b/docs/en/connector-v2/source/FakeSource.md @@ -1,4 +1,4 @@ -# LocalFile +# FakeSource > FakeSource connector