From db3b61c160d9734b47a9a20a4e772c45dbb0109c Mon Sep 17 00:00:00 2001 From: Casey Getz Date: Wed, 19 Jun 2019 17:18:51 -0700 Subject: [PATCH 1/2] Add configurable warm up for remote datacenter connections Previously, only local datacenter connections could be prewarmed in the router. This adds a new config to allow prewarming remote datacenter connections. This could be useful in deployments where cross-DC requests are likely. --- .../com.github.ambry/config/RouterConfig.java | 22 ++++++++++++++----- .../NonBlockingRouter.java | 22 +++++++++++++------ .../NonBlockingRouterTest.java | 10 ++++----- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java b/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java index cafbc4d325..55148df025 100644 --- a/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java +++ b/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java @@ -66,13 +66,23 @@ public class RouterConfig { /** * The percentage of {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortSsl} or - * {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortPlainText} to warm up in the startup. + * {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortPlainText} to warm up for data nodes in the local + * datacenter during startup. * {@link RouterConfig#routerConnectionsWarmUpTimeoutMs} may need to be adjusted. */ - @Config("router.connections.warm.up.percentage.per.port") + @Config("router.connections.local.dc.warm.up.percentage") @Default("25") - public final int routerConnectionsWarmUpPercentagePerPort; + public final int routerConnectionsLocalDcWarmUpPercentage; + /** + * The percentage of {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortSsl} or + * {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortPlainText} to warm up for data nodes in the local + * datacenter during startup. + * {@link RouterConfig#routerConnectionsWarmUpTimeoutMs} may need to be adjusted. + */ + @Config("router.connections.remote.dc.warm.up.percentage") + @Default("25") + public final int routerConnectionsRemoteDcWarmUpPercentage; /** * The max time allowed to establish connections to local DC in the startup */ @@ -300,8 +310,10 @@ public RouterConfig(VerifiableProperties verifiableProperties) { verifiableProperties.getIntInRange("router.scaling.unit.max.connections.per.port.plain.text", 5, 1, 100); routerScalingUnitMaxConnectionsPerPortSsl = verifiableProperties.getIntInRange("router.scaling.unit.max.connections.per.port.ssl", 2, 1, 100); - routerConnectionsWarmUpPercentagePerPort = - verifiableProperties.getIntInRange("router.connections.warm.up.percentage.per.port", 25, 0, 100); + routerConnectionsLocalDcWarmUpPercentage = + verifiableProperties.getIntInRange("router.connections.local.dc.warm.up.percentage", 25, 0, 100); + routerConnectionsRemoteDcWarmUpPercentage = + verifiableProperties.getIntInRange("router.connections.remote.dc.warm.up.percentage", 0, 0, 100); routerConnectionsWarmUpTimeoutMs = verifiableProperties.getIntInRange("router.connections.warm.up.timeout.ms", 5000, 0, Integer.MAX_VALUE); routerConnectionCheckoutTimeoutMs = diff --git a/ambry-router/src/main/java/com.github.ambry.router/NonBlockingRouter.java b/ambry-router/src/main/java/com.github.ambry.router/NonBlockingRouter.java index 44f26d8652..8db4177109 100644 --- a/ambry-router/src/main/java/com.github.ambry.router/NonBlockingRouter.java +++ b/ambry-router/src/main/java/com.github.ambry.router/NonBlockingRouter.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Future; import java.util.concurrent.ThreadLocalRandom; @@ -553,14 +554,21 @@ private class OperationController implements Runnable { */ OperationController(String suffix, String defaultPartitionClass, AccountService accountService) throws IOException { networkClient = networkClientFactory.getNetworkClient(); - // Warm up connections to dataNodes in local DC. + // Warm up connections to dataNodes in local and remote DCs. List responseInfos = new ArrayList<>(); - networkClient.warmUpConnections(clusterMap.getDataNodeIds() - .stream() - .filter(dataNodeId -> clusterMap.getDatacenterName(clusterMap.getLocalDatacenterId()) - .equals(dataNodeId.getDatacenterName())) - .collect(Collectors.toList()), routerConfig.routerConnectionsWarmUpPercentagePerPort, - routerConfig.routerConnectionsWarmUpTimeoutMs, responseInfos); + + String localDatacenter = clusterMap.getDatacenterName(clusterMap.getLocalDatacenterId()); + Map> localAndRemoteNodes = clusterMap.getDataNodeIds() + .stream() + .collect(Collectors.partitioningBy(dataNodeId -> localDatacenter.equals(dataNodeId.getDatacenterName()))); + logger.info("Warming up local datacenter connections to {} nodes", localAndRemoteNodes.get(true).size()); + networkClient.warmUpConnections(localAndRemoteNodes.get(true), + routerConfig.routerConnectionsLocalDcWarmUpPercentage, routerConfig.routerConnectionsWarmUpTimeoutMs, + responseInfos); + logger.info("Warming up remote datacenter connections to {} nodes", localAndRemoteNodes.get(false).size()); + networkClient.warmUpConnections(localAndRemoteNodes.get(false), + routerConfig.routerConnectionsRemoteDcWarmUpPercentage, routerConfig.routerConnectionsWarmUpTimeoutMs, + responseInfos); // Update ResponseHandler immediately if connections lost to certain nodes. for (ResponseInfo responseInfo : responseInfos) { if (responseInfo.getRequestInfo() == null) { diff --git a/ambry-router/src/test/java/com.github.ambry.router/NonBlockingRouterTest.java b/ambry-router/src/test/java/com.github.ambry.router/NonBlockingRouterTest.java index 78bbeb7559..9417e7a0f7 100644 --- a/ambry-router/src/test/java/com.github.ambry.router/NonBlockingRouterTest.java +++ b/ambry-router/src/test/java/com.github.ambry.router/NonBlockingRouterTest.java @@ -172,6 +172,8 @@ private Properties getNonBlockingRouterProperties(String routerDataCenter) { properties.setProperty("router.delete.success.target", Integer.toString(DELETE_SUCCESS_TARGET)); properties.setProperty("router.connection.checkout.timeout.ms", Integer.toString(CHECKOUT_TIMEOUT_MS)); properties.setProperty("router.request.timeout.ms", Integer.toString(REQUEST_TIMEOUT_MS)); + properties.setProperty("router.connections.local.dc.warm.up.percentage", Integer.toString(67)); + properties.setProperty("router.connections.remote.dc.warm.up.percentage", Integer.toString(34)); properties.setProperty("clustermap.cluster.name", "test"); properties.setProperty("clustermap.datacenter.name", "dc1"); properties.setProperty("clustermap.host.name", "localhost"); @@ -859,12 +861,8 @@ public void testWarmUpConnectionFailureHandling() throws IOException { MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap); mockSelectorState.set(MockSelectorState.FailConnectionInitiationOnPoll); setRouter(props, mockServerLayout, new LoggingNotificationSystem()); - List localNodes = mockClusterMap.getDataNodes() - .stream() - .filter(node -> node.getDatacenterName().equals("DC3")) - .collect(Collectors.toList()); - for (DataNodeId node : localNodes) { - assertTrue("Local node should be marked as timed out by ResponseHandler.", ((MockDataNodeId) node).isTimedOut()); + for (DataNodeId node : mockClusterMap.getDataNodes()) { + assertTrue("Node should be marked as timed out by ResponseHandler.", ((MockDataNodeId) node).isTimedOut()); } router.close(); mockSelectorState.set(MockSelectorState.Good); From 7044b1849f4734601fa2c83d23a0cac4c76853f9 Mon Sep 17 00:00:00 2001 From: Casey Getz Date: Thu, 20 Jun 2019 13:49:28 -0700 Subject: [PATCH 2/2] Address comments --- .../src/main/java/com.github.ambry/config/RouterConfig.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java b/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java index 55148df025..78f194579e 100644 --- a/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java +++ b/ambry-api/src/main/java/com.github.ambry/config/RouterConfig.java @@ -76,12 +76,12 @@ public class RouterConfig { /** * The percentage of {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortSsl} or - * {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortPlainText} to warm up for data nodes in the local - * datacenter during startup. + * {@link RouterConfig#routerScalingUnitMaxConnectionsPerPortPlainText} to warm up for data nodes in remote + * datacenters during startup. * {@link RouterConfig#routerConnectionsWarmUpTimeoutMs} may need to be adjusted. */ @Config("router.connections.remote.dc.warm.up.percentage") - @Default("25") + @Default("0") public final int routerConnectionsRemoteDcWarmUpPercentage; /** * The max time allowed to establish connections to local DC in the startup