Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extend JDBC URL pattern to support failover #411

Merged
merged 3 commits into from
Mar 5, 2022
Merged

Extend JDBC URL pattern to support failover #411

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6f02da0
support failover
loneylee Mar 3, 2022
f957507
fix pattern and add ut
loneylee Mar 4, 2022
747b00a
fix ut wrong pair host and port
loneylee Mar 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion clickhouse-native-jdbc/src/main/java/com/github/housepower/client/NativeClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,12 @@ public class NativeClient {
private static final Logger LOG = LoggerFactory.getLogger(NativeClient.class);

public static NativeClient connect(ClickHouseConfig configure) throws SQLException {
return connect(configure.host(), configure.port(), configure);
}

public static NativeClient connect(String host, int port, ClickHouseConfig configure) throws SQLException {
try {
SocketAddress endpoint = new InetSocketAddress(configure.host(), configure.port());
SocketAddress endpoint = new InetSocketAddress(host, port);
// TODO support proxy
Socket socket = new Socket();
socket.setTcpNoDelay(true);
Expand Down
52 changes: 50 additions & 2 deletions clickhouse-native-jdbc/src/main/java/com/github/housepower/jdbc/ClickHouseConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,17 @@

import javax.annotation.Nullable;
import java.net.InetSocketAddress;
import java.sql.*;
import java.sql.Array;
import java.sql.ClientInfoStatus;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLClientInfoException;
import java.sql.SQLException;
import java.sql.SQLWarning;
import java.sql.Statement;
import java.sql.Struct;
import java.time.Duration;
import java.time.ZoneId;
import java.util.HashMap;
Expand All @@ -45,6 +55,8 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.github.housepower.jdbc.ClickhouseJdbcUrlParser.PORT_DELIMITER;

public class ClickHouseConnection implements SQLConnection {

private static final Logger LOG = LoggerFactory.getLogger(ClickHouseConnection.class);
Expand Down Expand Up @@ -305,7 +317,43 @@ public static ClickHouseConnection createClickHouseConnection(ClickHouseConfig c
}

private static NativeContext createNativeContext(ClickHouseConfig configure) throws SQLException {
NativeClient nativeClient = NativeClient.connect(configure);
if (configure.hosts().size() == 1) {
NativeClient nativeClient = NativeClient.connect(configure);
return new NativeContext(clientContext(nativeClient, configure), serverContext(nativeClient, configure), nativeClient);
}

return createFailoverNativeContext(configure);
}

private static NativeContext createFailoverNativeContext(ClickHouseConfig configure) throws SQLException {
NativeClient nativeClient = null;
SQLException lastException = null;

int tryIndex = 0;
do {
String hostAndPort = configure.hosts().get(tryIndex);
String[] hostAndPortSplit = hostAndPort.split(PORT_DELIMITER, 2);
String host = hostAndPortSplit[0];
int port;

if (hostAndPortSplit.length == 2) {
port = Integer.parseInt(hostAndPortSplit[1]);
} else {
port = configure.port();
}

try {
nativeClient = NativeClient.connect(host, port, configure);
} catch (SQLException e) {
lastException = e;
}
tryIndex++;
} while (nativeClient == null && tryIndex < configure.hosts().size());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like the former node always has high priority, so it does not a load-balanced implementation? If yes, let's add clear comments on the regex pattern to avoid surprising users.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it does not a load-balanced implementation


if (nativeClient == null) {
throw lastException;
}

return new NativeContext(clientContext(nativeClient, configure), serverContext(nativeClient, configure), nativeClient);
}

Expand Down
123 changes: 49 additions & 74 deletions clickhouse-native-jdbc/src/main/java/com/github/housepower/jdbc/ClickhouseJdbcUrlParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,17 @@
package com.github.housepower.jdbc;

import com.github.housepower.exception.InvalidValueException;
import com.github.housepower.misc.Validate;
import com.github.housepower.settings.SettingKey;
import com.github.housepower.log.Logger;
import com.github.housepower.log.LoggerFactory;
import com.github.housepower.misc.Validate;
import com.github.housepower.settings.SettingKey;

import java.io.Serializable;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

Expand All @@ -32,92 +34,65 @@ public class ClickhouseJdbcUrlParser {
public static final String CLICKHOUSE_PREFIX = "clickhouse:";
public static final String JDBC_CLICKHOUSE_PREFIX = JDBC_PREFIX + CLICKHOUSE_PREFIX;

public static final Pattern DB_PATH_PATTERN = Pattern.compile("/([a-zA-Z0-9_]+)");
public static final Pattern HOST_PORT_PATH_PATTERN = Pattern.compile("//(?<host>[^/:\\s]+)(:(?<port>\\d+))?");
public static final String HOST_DELIMITER = ",";
public static final String PORT_DELIMITER = ":";

/**
* Jdbc Url sames like:
* '//[host1][:port1],[host2][:port2],[host3][:port3]]...[/[database]][?propertyName1=propertyValue1[&propertyName2=propertyValue2]...]'
*
* Default_port is used when port does not exist.
*/
public static final Pattern CONNECTION_PATTERN = Pattern.compile("//(?<hosts>([^/?:,\\s]+(:\\d+)?)(,[^/?:,\\s]+(:\\d+)?)*)" // hosts: required; starts with "//" followed by any char except "/", "?"
+ "(?:/(?<database>([a-zA-Z0-9_]+)))?" // database: optional; starts with "/", and then followed by any char except "?"
+ "(?:\\?(?<properties>.*))?"); // properties: optional; starts with "?", and then followed by any char

private static final Logger LOG = LoggerFactory.getLogger(ClickhouseJdbcUrlParser.class);

public static Map<SettingKey, Serializable> parseJdbcUrl(String jdbcUrl) {
try {
URI uri = new URI(jdbcUrl.substring(JDBC_PREFIX.length()));
String host = parseHost(jdbcUrl);
Integer port = parsePort(jdbcUrl);
String database = parseDatabase(jdbcUrl);
Map<SettingKey, Serializable> settings = new HashMap<>();
settings.put(SettingKey.host, host);
settings.put(SettingKey.port, port);
settings.put(SettingKey.database, database);
settings.putAll(extractQueryParameters(uri.getQuery()));

return settings;
} catch (URISyntaxException ex) {
throw new InvalidValueException(ex);
String uri = jdbcUrl.substring(JDBC_CLICKHOUSE_PREFIX.length());
Matcher matcher = CONNECTION_PATTERN.matcher(uri);
if (!matcher.matches()) {
throw new InvalidValueException("Connection is not support");
}
}

public static Map<SettingKey, Serializable> parseProperties(Properties properties) {
Map<SettingKey, Serializable> settings = new HashMap<>();

for (String name : properties.stringPropertyNames()) {
String value = properties.getProperty(name);
String hosts = matcher.group("hosts");
String database = matcher.group("database");
String properties = matcher.group("properties");

parseSetting(settings, name, value);
}
if (hosts.contains(HOST_DELIMITER)) { // multi-host
settings.put(SettingKey.host, hosts);
} else { // standard-host
String[] hostAndPort = hosts.split(PORT_DELIMITER, 2);

return settings;
}
settings.put(SettingKey.host, hostAndPort[0]);

private static String parseDatabase(String jdbcUrl) throws URISyntaxException {
URI uri = new URI(jdbcUrl.substring(JDBC_PREFIX.length()));
String database = uri.getPath();
if (database != null && !database.isEmpty()) {
Matcher m = DB_PATH_PATTERN.matcher(database);
if (m.matches()) {
database = m.group(1);
} else {
throw new URISyntaxException("wrong database name path: '" + database + "'", jdbcUrl);
if (hostAndPort.length == 2) {
if (Integer.parseInt(hostAndPort[1]) == 8123) {
LOG.warn("8123 is default HTTP port, you may connect with error protocol!");
}
settings.put(SettingKey.port, Integer.parseInt(hostAndPort[1]));
}
}
if (database != null && database.isEmpty()) {
database = "default";
}
return database;
}

private static String parseHost(String jdbcUrl) throws URISyntaxException {
String uriStr = jdbcUrl.substring(JDBC_PREFIX.length());
URI uri = new URI(uriStr);
String host = uri.getHost();
if (host == null || host.isEmpty()) {
Matcher m = HOST_PORT_PATH_PATTERN.matcher(uriStr);
if (m.find()) {
host = m.group("host");
} else {
throw new URISyntaxException("No valid host was found", jdbcUrl);
}
}
return host;
settings.put(SettingKey.database, database);
settings.putAll(extractQueryParameters(properties));

return settings;
}

private static int parsePort(String jdbcUrl) {
String uriStr = jdbcUrl.substring(JDBC_PREFIX.length());
URI uri;
try {
uri = new URI(uriStr);
} catch (Exception ex) {
throw new InvalidValueException(ex);
}
int port = uri.getPort();
if (port <= -1) {
Matcher m = HOST_PORT_PATH_PATTERN.matcher(uriStr);
if (m.find() && m.group("port") != null) {
port = Integer.parseInt(m.group("port"));
}
}
if (port == 8123) {
LOG.warn("8123 is default HTTP port, you may connect with error protocol!");
public static Map<SettingKey, Serializable> parseProperties(Properties properties) {
Map<SettingKey, Serializable> settings = new HashMap<>();

for (String name : properties.stringPropertyNames()) {
String value = properties.getProperty(name);

parseSetting(settings, name, value);
}
return port;

return settings;
}

public static Map<SettingKey, Serializable> extractQueryParameters(String queryParameters) {
Expand Down
19 changes: 18 additions & 1 deletion clickhouse-native-jdbc/src/main/java/com/github/housepower/settings/ClickHouseConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,21 @@
import java.io.Serializable;
import java.nio.charset.Charset;
import java.time.Duration;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;

import static com.github.housepower.jdbc.ClickhouseJdbcUrlParser.HOST_DELIMITER;

@Immutable
public class ClickHouseConfig implements Serializable {


private final String host;
private final List<String> hosts;
private final int port;
private final String database;
private final String user;
Expand All @@ -46,6 +52,7 @@ private ClickHouseConfig(String host, int port, String database, String user, St
Duration queryTimeout, Duration connectTimeout, boolean tcpKeepAlive,
String charset, String clientName, Map<SettingKey, Serializable> settings) {
this.host = host;
this.hosts = Arrays.asList(host.split(HOST_DELIMITER));
this.port = port;
this.database = database;
this.user = user;
Expand All @@ -62,6 +69,10 @@ public String host() {
return this.host;
}

public List<String> hosts() {
return this.hosts;
}

public int port() {
return this.port;
}
Expand Down Expand Up @@ -96,7 +107,13 @@ public String clientName() {

public String jdbcUrl() {
StringBuilder builder = new StringBuilder(ClickhouseJdbcUrlParser.JDBC_CLICKHOUSE_PREFIX)
.append("//").append(host).append(":").append(port).append("/").append(database)
.append("//").append(host);

if (hosts.size() == 1) {
builder.append(":").append(port);
}

builder.append("/").append(database)
.append("?").append(SettingKey.query_timeout.name()).append("=").append(queryTimeout.getSeconds())
.append("&").append(SettingKey.connect_timeout.name()).append("=").append(connectTimeout.getSeconds())
.append("&").append(SettingKey.charset.name()).append("=").append(charset)
Expand Down
Loading