Skip to content

Commit

Permalink
update all test for fakeData
Browse files Browse the repository at this point in the history
  • Loading branch information
tangjiafu committed Aug 21, 2022
1 parent f7f767a commit f5f9f20
Show file tree
Hide file tree
Showing 25 changed files with 721 additions and 466 deletions.
6 changes: 3 additions & 3 deletions ...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/fake/source/FakeRandomData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
if (BOOLEAN_TYPE.equals(fieldType)) {
return RandomUtils.nextInt(0, 2) == 1;
} else if (BYTE_TYPE.equals(fieldType)) {
return (byte) RandomUtils.nextInt(0, Byte.MAX_VALUE);
return (byte) RandomUtils.nextInt(Byte.MIN_VALUE, Byte.MAX_VALUE);
} else if (SHORT_TYPE.equals(fieldType)) {
return (short) RandomUtils.nextInt(Byte.MAX_VALUE, Short.MAX_VALUE);
} else if (INT_TYPE.equals(fieldType)) {
Expand All @@ -94,7 +94,7 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
} else if (fieldType instanceof DecimalType) {
DecimalType decimalType = (DecimalType) fieldType;
return new BigDecimal(RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()) + "." +
RandomStringUtils.randomNumeric(decimalType.getPrecision() - decimalType.getScale()));
RandomStringUtils.randomNumeric(decimalType.getScale()));
} else if (fieldType instanceof ArrayType) {
ArrayType<?, ?> arrayType = (ArrayType<?, ?>) fieldType;
BasicType<?> elementType = arrayType.getElementType();
Expand All @@ -116,7 +116,7 @@ private Object randomColumnValue(SeaTunnelDataType<?> fieldType) {
} else if (VOID_TYPE.equals(fieldType) || fieldType == null) {
return Void.TYPE;
} else {
throw new IllegalStateException("Unexpected value: " + fieldType);
throw new UnsupportedOperationException("Unexpected value: " + fieldType);
}
}

Expand Down
104 changes: 53 additions & 51 deletions ...e/seatunnel-flink-connector-v2-e2e/src/test/resources/assertion/fakesource_to_assert.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,64 +25,66 @@ env {
#execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}

# Docoment-Link
# https://seatunnel.apache.org/docs/category/source-v2
# https://seatunnel.apache.org/docs/category/sink-v2
# https://seatunnel.apache.org/docs/category/transform

source {
# This is a example source plugin **only for test and demonstrate the feature source plugin**
FakeSource {
result_table_name = "fake"
field_name = "name,age"
FakeSource {
fields {
c_map = "map<string, string>"
c_array = "array<tinyint>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_decimal = "decimal(30, 8)"
c_null = "null"
c_bytes = bytes
c_date = date
c_time = time
c_timestamp = timestamp
}

# If you would like to get more information about how to configure seatunnel and see full list of source plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
result_table_name = "fake"
}
}

transform {
sql {
sql = "select name,age from fake"
}

# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
transform {
sql {
sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake"
result_table_name = "sql"
}
}

sink {
Assert {
rules =
[{
field_name = name
field_type = string
field_value = [
{
rule_type = NOT_NULL
},
{
rule_type = MIN_LENGTH
rule_value = 3
},
{
rule_type = MAX_LENGTH
rule_value = 20
}
]
},{
field_name = age
field_type = int
field_value = [
{
rule_type = NOT_NULL
},
{
rule_type = MIN
rule_value = 1
},
{
rule_type = MAX
rule_value = 100
}
]
}
]
}
Assert {
rules = [
{
field_name = c_boolean
field_type = boolean
field_value = [
{
rule_type = NOT_NULL
}
]
},
{
field_name = c_string
field_type = string
field_value = [
{
rule_type = NOT_NULL
}
]
}
]
}
# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/Assert
}
39 changes: 24 additions & 15 deletions ...l-e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/fake/fakesource_to_console.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,37 @@ env {
}

source {
# This is a example source plugin **only for test and demonstrate the feature source plugin**
FakeSource {
result_table_name = "fake"
field_name = "name,age"
FakeSource {
fields {
c_map = "map<string, string>"
c_array = "array<tinyint>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_decimal = "decimal(30, 8)"
c_null = "null"
c_bytes = bytes
c_date = date
c_time = time
c_timestamp = timestamp
}

# If you would like to get more information about how to configure seatunnel and see full list of source plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/source-plugins/Fake
result_table_name = "fake"
}
}

transform {
sql {
sql = "select name,age from fake"
}
sql {
sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake"
result_table_name = "sql"
}

# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/transform-plugins/Sql
}

sink {
Console {}

# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/flink/configuration/sink-plugins/Console
}
53 changes: 33 additions & 20 deletions ...e2e/seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_json.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,38 +28,51 @@ env {

source {
FakeSource {
fields {
c_map = "map<string, string>"
c_array = "array<tinyint>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_decimal = "decimal(30, 8)"
c_null = "null"
c_bytes = bytes
c_date = date
c_time = time
c_timestamp = timestamp
}
result_table_name = "fake"
field_name = "name,age"
}

# If you would like to get more information about how to configure seatunnel and see full list of source plugins,
# please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}


transform {
sql {
sql = "select name,age from fake"
sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake"
result_table_name = "sql"
}

# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/transform/sql
}

sink {
HdfsFile {
path="/tmp/hive/warehouse/test2"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
file_format="json"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
path = "/tmp/hive/warehouse/test2"
row_delimiter = "\n"
partition_by = ["c_bigint"]
partition_dir_expression = "${k0}=${v0}"
is_partition_field_write_in_file = true
file_name_expression = "${transactionId}_${now}"
file_format = "json"
sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"]
filename_time_format = "yyyy.MM.dd"
is_enable_transaction = true
save_mode = "error"
}

# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile
}
55 changes: 34 additions & 21 deletions .../seatunnel-flink-connector-v2-e2e/src/test/resources/file/fakesource_to_hdfs_parquet.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,39 +28,52 @@ env {

source {
FakeSource {
fields {
c_map = "map<string, string>"
c_array = "array<tinyint>"
c_string = string
c_boolean = boolean
c_tinyint = tinyint
c_smallint = smallint
c_int = int
c_bigint = bigint
c_float = float
c_double = double
c_decimal = "decimal(30, 8)"
c_null = "null"
c_bytes = bytes
c_date = date
c_time = time
c_timestamp = timestamp
}
result_table_name = "fake"
field_name = "name,age"
}

# If you would like to get more information about how to configure seatunnel and see full list of source plugins,
# please go to https://seatunnel.apache.org/docs/connector-v2/source/Fake
}


transform {
sql {
sql = "select name,age from fake"
sql = "select c_string,c_boolean,c_tinyint,c_smallint,c_int,c_bigint,c_float,c_double from fake"
result_table_name = "sql"
}

# If you would like to get more information about how to configure seatunnel and see full list of transform plugins,
# please go to https://seatunnel.apache.org/docs/transform/sql
}

sink {
HdfsFile {
path="/tmp/hive/warehouse/test2"
field_delimiter="\t"
row_delimiter="\n"
partition_by=["age"]
partition_dir_expression="${k0}=${v0}"
is_partition_field_write_in_file=true
file_name_expression="${transactionId}_${now}"
file_format="parquet"
sink_columns=["name","age"]
filename_time_format="yyyy.MM.dd"
is_enable_transaction=true
save_mode="error"
path = "/tmp/hive/warehouse/test2"
field_delimiter = "\t"
row_delimiter = "\n"
partition_by = ["c_bigint"]
partition_dir_expression = "${k0}=${v0}"
is_partition_field_write_in_file = true
file_name_expression = "${transactionId}_${now}"
file_format = "parquet"
sink_columns = ["c_string", "c_boolean", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_float", "c_double"]
filename_time_format = "yyyy.MM.dd"
is_enable_transaction = true
save_mode = "error"
}

# If you would like to get more information about how to configure seatunnel and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/File
# please go to https://seatunnel.apache.org/docs/connector-v2/sink/HdfsFile
}
Loading

0 comments on commit f5f9f20

Please sign in to comment.