forked from apache/seatunnel
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Feature][Connector-V2] add tdengine source (apache#2832)
* [Feature][Connector-V2]add tdengine source and sink 1. add ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY to statement 2. add tdengine e2e module * SourceSplitEnumerator.Context Co-authored-by: bjyflihongyu <lihongyuinfo@jd.com> Co-authored-by: tyrantlucifer <tyrantlucifer@gmail.com>
- Loading branch information
Showing
21 changed files
with
1,598 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
# TDengine | ||
|
||
> TDengine sink connector | ||
## Description | ||
|
||
Used to write data to TDengine. You need to create stable before running seatunnel task | ||
|
||
## Key features | ||
|
||
- [x] [exactly-once](../../concept/connector-v2-features.md) | ||
- [ ] [cdc](../../concept/connector-v2-features.md) | ||
|
||
## Options | ||
|
||
| name | type | required | default value | | ||
|----------------------------|---------|----------|---------------| | ||
| url | string | yes | - | | ||
| username | string | yes | - | | ||
| password | string | yes | - | | ||
| database | string | yes | | | ||
| stable | string | yes | - | | ||
| timezone | string | no | UTC | | ||
|
||
### url [string] | ||
|
||
the url of the TDengine when you select the TDengine | ||
|
||
e.g. | ||
``` | ||
jdbc:TAOS-RS://localhost:6041/ | ||
``` | ||
|
||
### username [string] | ||
|
||
the username of the TDengine when you select | ||
|
||
### password [string] | ||
|
||
the password of the TDengine when you select | ||
|
||
### database [string] | ||
|
||
the database of the TDengine when you select | ||
|
||
### stable [string] | ||
|
||
the stable of the TDengine when you select | ||
|
||
### timezone [string] | ||
|
||
the timeznoe of the TDengine sever, it's important to the ts field | ||
|
||
## Example | ||
|
||
### sink | ||
|
||
```hocon | ||
sink { | ||
TDengine { | ||
url : "jdbc:TAOS-RS://localhost:6041/" | ||
username : "root" | ||
password : "taosdata" | ||
database : "power2" | ||
stable : "meters2" | ||
timezone: UTC | ||
} | ||
} | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,82 @@ | ||
# TDengine | ||
|
||
> TDengine source connector | ||
## Description | ||
|
||
Read external data source data through TDengine. | ||
|
||
## Key features | ||
|
||
- [x] [batch](../../concept/connector-v2-features.md) | ||
- [ ] [stream](../../concept/connector-v2-features.md) | ||
- [x] [exactly-once](../../concept/connector-v2-features.md) | ||
- [ ] [column projection](../../concept/connector-v2-features.md) | ||
|
||
supports query SQL and can achieve projection effect. | ||
|
||
- [x] [parallelism](../../concept/connector-v2-features.md) | ||
- [ ] [support user-defined split](../../concept/connector-v2-features.md) | ||
|
||
## Options | ||
|
||
| name | type | required | default value | | ||
|----------------------------|---------|----------|---------------| | ||
| url | string | yes | - | | ||
| username | string | yes | - | | ||
| password | string | yes | - | | ||
| database | string | yes | | | ||
| stable | string | yes | - | | ||
| lower_bound | long | yes | - | | ||
| upper_bound | long | yes | - | | ||
|
||
### url [string] | ||
|
||
the url of the TDengine when you select the TDengine | ||
|
||
e.g. | ||
``` | ||
jdbc:TAOS-RS://localhost:6041/ | ||
``` | ||
|
||
### username [string] | ||
|
||
the username of the TDengine when you select | ||
|
||
### password [string] | ||
|
||
the password of the TDengine when you select | ||
|
||
### database [string] | ||
|
||
the database of the TDengine when you select | ||
|
||
### stable [string] | ||
|
||
the stable of the TDengine when you select | ||
|
||
### lower_bound [long] | ||
|
||
the lower_bound of the migration period | ||
|
||
### upper_bound [long] | ||
|
||
the upper_bound of the migration period | ||
|
||
## Example | ||
|
||
### source | ||
```hocon | ||
source { | ||
TDengine { | ||
url : "jdbc:TAOS-RS://localhost:6041/" | ||
username : "root" | ||
password : "taosdata" | ||
database : "power" | ||
stable : "meters" | ||
lower_bound : "2018-10-03 14:38:05.000" | ||
upper_bound : "2018-10-03 14:38:16.800" | ||
result_table_name = "tdengine_result" | ||
} | ||
} | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,45 @@ | ||
<?xml version="1.0" encoding="UTF-8"?> | ||
<!-- | ||
Licensed to the Apache Software Foundation (ASF) under one or more | ||
contributor license agreements. See the NOTICE file distributed with | ||
this work for additional information regarding copyright ownership. | ||
The ASF licenses this file to You under the Apache License, Version 2.0 | ||
(the "License"); you may not use this file except in compliance with | ||
the License. You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
--> | ||
<project xmlns="http://maven.apache.org/POM/4.0.0" | ||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
<parent> | ||
<groupId>org.apache.seatunnel</groupId> | ||
<artifactId>seatunnel-connectors-v2</artifactId> | ||
<version>${revision}</version> | ||
</parent> | ||
<modelVersion>4.0.0</modelVersion> | ||
|
||
<artifactId>connector-tdengine</artifactId> | ||
|
||
<dependencies> | ||
<dependency> | ||
<groupId>org.apache.seatunnel</groupId> | ||
<artifactId>connector-common</artifactId> | ||
<version>${project.version}</version> | ||
</dependency> | ||
<dependency> | ||
<groupId>com.taosdata.jdbc</groupId> | ||
<artifactId>taos-jdbcdriver</artifactId> | ||
<version>3.0.3</version> | ||
</dependency> | ||
</dependencies> | ||
|
||
</project> |
79 changes: 79 additions & 0 deletions
79
.../java/org/apache/seatunnel/connectors/seatunnel/tdengine/config/TDengineSourceConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,79 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.seatunnel.connectors.seatunnel.tdengine.config; | ||
|
||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.DATABASE; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.LOWER_BOUND; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.PASSWORD; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.STABLE; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.TIMEZONE; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.UPPER_BOUND; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.URL; | ||
import static org.apache.seatunnel.connectors.seatunnel.tdengine.config.TDengineSourceConfig.ConfigNames.USERNAME; | ||
|
||
import org.apache.seatunnel.shade.com.typesafe.config.Config; | ||
|
||
import lombok.Data; | ||
|
||
import java.io.Serializable; | ||
import java.util.List; | ||
|
||
@Data | ||
public class TDengineSourceConfig implements Serializable { | ||
|
||
/** | ||
* jdbc:TAOS-RS://localhost:6041/ | ||
*/ | ||
private String url; | ||
private String username; | ||
private String password; | ||
private String database; | ||
private String stable; | ||
//param of timezone in 'jdbc:TAOS-RS' just effect on taosadapter side, other than the JDBC client side | ||
//so this param represent the server-side timezone setting up | ||
private String timezone; | ||
private String lowerBound; | ||
private String upperBound; | ||
private List<String> fields; | ||
private List<String> tags; | ||
|
||
public static TDengineSourceConfig buildSourceConfig(Config pluginConfig) { | ||
TDengineSourceConfig tdengineSourceConfig = new TDengineSourceConfig(); | ||
tdengineSourceConfig.setUrl(pluginConfig.hasPath(URL) ? pluginConfig.getString(URL) : null); | ||
tdengineSourceConfig.setDatabase(pluginConfig.hasPath(DATABASE) ? pluginConfig.getString(DATABASE) : null); | ||
tdengineSourceConfig.setStable(pluginConfig.hasPath(STABLE) ? pluginConfig.getString(STABLE) : null); | ||
tdengineSourceConfig.setUsername(pluginConfig.hasPath(USERNAME) ? pluginConfig.getString(USERNAME) : null); | ||
tdengineSourceConfig.setPassword(pluginConfig.hasPath(PASSWORD) ? pluginConfig.getString(PASSWORD) : null); | ||
tdengineSourceConfig.setUpperBound(pluginConfig.hasPath(UPPER_BOUND) ? pluginConfig.getString(UPPER_BOUND) : null); | ||
tdengineSourceConfig.setLowerBound(pluginConfig.hasPath(LOWER_BOUND) ? pluginConfig.getString(LOWER_BOUND) : null); | ||
tdengineSourceConfig.setTimezone(pluginConfig.hasPath(TIMEZONE) ? pluginConfig.getString(TIMEZONE) : "UTC"); | ||
return tdengineSourceConfig; | ||
} | ||
|
||
public static class ConfigNames { | ||
|
||
public static String URL = "url"; | ||
public static String USERNAME = "username"; | ||
public static String PASSWORD = "password"; | ||
public static String DATABASE = "database"; | ||
public static String STABLE = "stable"; | ||
public static String TIMEZONE = "timezone"; | ||
public static String LOWER_BOUND = "lower_bound"; | ||
public static String UPPER_BOUND = "upper_bound"; | ||
} | ||
} |
36 changes: 36 additions & 0 deletions
36
.../apache/seatunnel/connectors/seatunnel/tdengine/exception/TDengineConnectorException.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one | ||
* or more contributor license agreements. See the NOTICE file | ||
* distributed with this work for additional information | ||
* regarding copyright ownership. The ASF licenses this file | ||
* to you under the Apache License, Version 2.0 (the | ||
* "License"); you may not use this file except in compliance | ||
* with the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.seatunnel.connectors.seatunnel.tdengine.exception; | ||
|
||
import org.apache.seatunnel.common.exception.SeaTunnelErrorCode; | ||
import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; | ||
|
||
public class TDengineConnectorException extends SeaTunnelRuntimeException { | ||
public TDengineConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage) { | ||
super(seaTunnelErrorCode, errorMessage); | ||
} | ||
|
||
public TDengineConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, String errorMessage, Throwable cause) { | ||
super(seaTunnelErrorCode, errorMessage, cause); | ||
} | ||
|
||
public TDengineConnectorException(SeaTunnelErrorCode seaTunnelErrorCode, Throwable cause) { | ||
super(seaTunnelErrorCode, cause); | ||
} | ||
} |
63 changes: 63 additions & 0 deletions
63
...e/src/main/java/org/apache/seatunnel/connectors/seatunnel/tdengine/sink/TDengineSink.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
/* | ||
* Licensed to the Apache Software Foundation (ASF) under one or more | ||
* contributor license agreements. See the NOTICE file distributed with | ||
* this work for additional information regarding copyright ownership. | ||
* The ASF licenses this file to You under the Apache License, Version 2.0 | ||
* (the "License"); you may not use this file except in compliance with | ||
* the License. You may obtain a copy of the License at | ||
* | ||
* http://www.apache.org/licenses/LICENSE-2.0 | ||
* | ||
* Unless required by applicable law or agreed to in writing, software | ||
* distributed under the License is distributed on an "AS IS" BASIS, | ||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
* See the License for the specific language governing permissions and | ||
* limitations under the License. | ||
*/ | ||
|
||
package org.apache.seatunnel.connectors.seatunnel.tdengine.sink; | ||
|
||
import org.apache.seatunnel.api.sink.SeaTunnelSink; | ||
import org.apache.seatunnel.api.sink.SinkWriter; | ||
import org.apache.seatunnel.api.table.type.SeaTunnelDataType; | ||
import org.apache.seatunnel.api.table.type.SeaTunnelRow; | ||
import org.apache.seatunnel.api.table.type.SeaTunnelRowType; | ||
import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink; | ||
import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; | ||
|
||
import org.apache.seatunnel.shade.com.typesafe.config.Config; | ||
|
||
import com.google.auto.service.AutoService; | ||
|
||
import java.io.IOException; | ||
|
||
@AutoService(SeaTunnelSink.class) | ||
public class TDengineSink extends AbstractSimpleSink<SeaTunnelRow, Void> { | ||
private SeaTunnelRowType seaTunnelRowType; | ||
private Config pluginConfig; | ||
|
||
@Override | ||
public void setTypeInfo(SeaTunnelRowType seaTunnelRowType) { | ||
this.seaTunnelRowType = seaTunnelRowType; | ||
} | ||
|
||
@Override | ||
public SeaTunnelDataType<SeaTunnelRow> getConsumedType() { | ||
return this.seaTunnelRowType; | ||
} | ||
|
||
@Override | ||
public AbstractSinkWriter<SeaTunnelRow, Void> createWriter(SinkWriter.Context context) throws IOException { | ||
return new TDengineSinkWriter(pluginConfig, seaTunnelRowType); | ||
} | ||
|
||
@Override | ||
public void prepare(Config pluginConfig) { | ||
this.pluginConfig = pluginConfig; | ||
} | ||
|
||
@Override | ||
public String getPluginName() { | ||
return "TDengine"; | ||
} | ||
} |
Oops, something went wrong.