diff --git a/SingularityClient/pom.xml b/SingularityClient/pom.xml
index 1bad3e101e..fef6f7b2e4 100644
--- a/SingularityClient/pom.xml
+++ b/SingularityClient/pom.xml
@@ -21,6 +21,11 @@
guava
+
+ com.github.rholder
+ guava-retrying
+
+
org.slf4j
slf4j-api
@@ -77,6 +82,18 @@
test
+
+ org.assertj
+ assertj-core
+ test
+
+
+
+ org.mockito
+ mockito-core
+ test
+
+
org.slf4j
slf4j-simple
diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java
index 8db975806e..cf511b9f52 100644
--- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java
+++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java
@@ -2,13 +2,18 @@
import static com.google.common.base.Preconditions.checkNotNull;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Function;
+import java.util.function.Predicate;
import javax.inject.Provider;
@@ -16,6 +21,11 @@
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.github.rholder.retry.RetryException;
+import com.github.rholder.retry.Retryer;
+import com.github.rholder.retry.RetryerBuilder;
+import com.github.rholder.retry.StopStrategies;
+import com.github.rholder.retry.WaitStrategies;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -26,6 +36,7 @@
import com.hubspot.horizon.HttpRequest;
import com.hubspot.horizon.HttpRequest.Method;
import com.hubspot.horizon.HttpResponse;
+import com.hubspot.horizon.RetryStrategy;
import com.hubspot.mesos.json.MesosFileChunkObject;
import com.hubspot.singularity.ExtendedTaskState;
import com.hubspot.singularity.MachineState;
@@ -210,10 +221,12 @@ public class SingularityClient {
private final HttpClient httpClient;
private final Optional credentials;
+ private final Retryer httpResponseRetryer;
+
@Inject
@Deprecated
public SingularityClient(@Named(SingularityClientModule.CONTEXT_PATH) String contextPath, @Named(SingularityClientModule.HTTP_CLIENT_NAME) HttpClient httpClient, @Named(SingularityClientModule.HOSTS_PROPERTY_NAME) String hosts) {
- this(contextPath, httpClient, Arrays.asList(hosts.split(",")), Optional.absent());
+ this(contextPath, httpClient, Arrays.asList(hosts.split(",")), Optional.absent());
}
public SingularityClient(String contextPath, HttpClient httpClient, List hosts, Optional credentials) {
@@ -229,6 +242,10 @@ public SingularityClient(String contextPath, HttpClient httpClient, List
}
public SingularityClient(String contextPath, HttpClient httpClient, Provider> hostsProvider, Optional credentials, boolean ssl) {
+ this(contextPath, httpClient, hostsProvider, credentials, ssl, 3, HttpResponse::isServerError);
+ }
+
+ public SingularityClient(String contextPath, HttpClient httpClient, Provider> hostsProvider, Optional credentials, boolean ssl, int retryAttempts, Predicate retryStrategy) {
this.httpClient = httpClient;
this.contextPath = contextPath;
@@ -237,15 +254,17 @@ public SingularityClient(String contextPath, HttpClient httpClient, ProvidernewBuilder()
+ .withStopStrategy(StopStrategies.stopAfterAttempt(retryAttempts))
+ .withWaitStrategy(WaitStrategies.exponentialWait())
+ .retryIfResult(retryStrategy::test)
+ .retryIfException()
+ .build();
}
- private String getHost() {
- final List hosts = hostsProvider.get();
- return hosts.get(random.nextInt(hosts.size()));
+ private String getApiBase(String host) {
+ return String.format(BASE_API_FORMAT, ssl ? "https" : "http", host, contextPath);
}
//
@@ -278,13 +297,13 @@ private SingularityClientException fail(String type, HttpResponse response) {
throw new SingularityClientException(String.format("Failed '%s' action on Singularity (%s) - code: %s, %s", type, uri, response.getStatusCode(), body), response.getStatusCode());
}
- private Optional getSingle(String uri, String type, String id, Class clazz) {
- return getSingleWithParams(uri, type, id, Optional.absent(), clazz);
+ private Optional getSingle(Function hostToUrl, String type, String id, Class clazz) {
+ return getSingleWithParams(hostToUrl, type, id, Optional.absent(), clazz);
}
- private Optional getSingleWithParams(String uri, String type, String id, Optional