The zoo is under new management (zk cleanup)

SingularityBase/src/main/java/com/hubspot/deploy/HealthcheckOptions.java

@@ -9,6 +9,7 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
+import com.hubspot.singularity.HealthcheckMethod;
 import com.hubspot.singularity.HealthcheckProtocol;
 
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -20,6 +21,7 @@ public class HealthcheckOptions {
   private final Optional<Integer> portIndex;
   private final Optional<Long> portNumber;
   private final Optional<HealthcheckProtocol> protocol;
+  private final Optional<HealthcheckMethod> method;
   private final Optional<Integer> startupTimeoutSeconds;
   private final Optional<Integer> startupDelaySeconds;
   private final Optional<Integer> startupIntervalSeconds;
@@ -34,6 +36,7 @@ public HealthcheckOptions(@JsonProperty("uri") Optional<String> uri,
                             @JsonProperty("portIndex") Optional<Integer> portIndex,
                             @JsonProperty("portNumber") Optional<Long> portNumber,
                             @JsonProperty("protocol") Optional<HealthcheckProtocol> protocol,
+                            @JsonProperty("method") Optional<HealthcheckMethod> method,
                             @JsonProperty("startupTimeoutSeconds") Optional<Integer> startupTimeoutSeconds,
                             @JsonProperty("startupDelaySeconds") Optional<Integer> startupDelaySeconds,
                             @JsonProperty("startupIntervalSeconds") Optional<Integer> startupIntervalSeconds,
@@ -46,6 +49,7 @@ public HealthcheckOptions(@JsonProperty("uri") Optional<String> uri,
     this.portIndex = portIndex;
     this.portNumber = portNumber;
     this.protocol = protocol;
+    this.method = method;
     this.startupTimeoutSeconds = startupTimeoutSeconds;
     this.startupDelaySeconds = startupDelaySeconds;
     this.startupIntervalSeconds = startupIntervalSeconds;
@@ -59,7 +63,9 @@ public HealthcheckOptions(@JsonProperty("uri") Optional<String> uri,
   public HealthcheckOptions(String uri,
                             Optional<Integer> portIndex,
                             Optional<Long> portNumber,
-                            Optional<HealthcheckProtocol> protocol,Optional<Integer> startupTimeoutSeconds,
+                            Optional<HealthcheckProtocol> protocol,
+                            Optional<HealthcheckMethod> method,
+                            Optional<Integer> startupTimeoutSeconds,
                             Optional<Integer> startupDelaySeconds,
                             Optional<Integer> startupIntervalSeconds,
                             Optional<Integer> intervalSeconds,
@@ -71,6 +77,7 @@ public HealthcheckOptions(String uri,
     this.portIndex = portIndex;
     this.portNumber = portNumber;
     this.protocol = protocol;
+    this.method = method;
     this.startupTimeoutSeconds = startupTimeoutSeconds;
     this.startupDelaySeconds = startupDelaySeconds;
     this.startupIntervalSeconds = startupIntervalSeconds;
@@ -87,6 +94,7 @@ public HealthcheckOptionsBuilder toBuilder() {
       .setPortIndex(portIndex)
       .setPortNumber(portNumber)
       .setProtocol(protocol)
+      .setMethod(method)
       .setStartupTimeoutSeconds(startupTimeoutSeconds)
       .setStartupDelaySeconds(startupDelaySeconds)
       .setStartupIntervalSeconds(startupIntervalSeconds)
@@ -102,7 +110,7 @@ public Optional<String> getUri() {
     return uri;
   }
 
-  @Schema(description = "Perform healthcheck on this dynamically allocated port (e.g. 0 for first port), defaults to first port")
+  @Schema(description = "Perform healthcheck on this dynamically allocated port (e.g. 0 for first port); defaults to first port")
   public Optional<Integer> getPortIndex() {
     return portIndex;
   }
@@ -112,11 +120,16 @@ public Optional<Long> getPortNumber() {
     return portNumber;
   }
 
-  @Schema(description = "Healthcheck protocol - HTTP or HTTPS")
+  @Schema(description = "Healthcheck protocol - HTTP or HTTPS for HTTP/1, HTTP2 or HTTPS2 for HTTP/2")
   public Optional<HealthcheckProtocol> getProtocol() {
     return protocol;
   }
 
+  @Schema(description ="Healthcheck HTTP method - GET or POST; GET by default")
+  public Optional<HealthcheckMethod> getMethod() {
+    return method;
+  }
+
   @Schema(description = "Consider the task unhealthy/failed if the app has not started responding to healthchecks in this amount of time")
   public Optional<Integer> getStartupTimeoutSeconds() {
     return startupTimeoutSeconds;
@@ -170,6 +183,7 @@ public boolean equals(Object o) {
         Objects.equals(portIndex, that.portIndex) &&
         Objects.equals(portNumber, that.portNumber) &&
         Objects.equals(protocol, that.protocol) &&
+        Objects.equals(method, that.method) &&
         Objects.equals(startupTimeoutSeconds, that.startupTimeoutSeconds) &&
         Objects.equals(startupDelaySeconds, that.startupDelaySeconds) &&
         Objects.equals(startupIntervalSeconds, that.startupIntervalSeconds) &&
@@ -182,7 +196,7 @@ public boolean equals(Object o) {
 
   @Override
   public int hashCode() {
-    return Objects.hash(uri, portIndex, portNumber, protocol, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
+    return Objects.hash(uri, portIndex, portNumber, protocol, method, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
   }
 
   @Override
@@ -192,6 +206,7 @@ public String toString() {
         ", portIndex=" + portIndex +
         ", portNumber=" + portNumber +
         ", protocol=" + protocol +
+        ", method=" + method +
         ", startupTimeoutSeconds=" + startupTimeoutSeconds +
         ", startupDelaySeconds=" + startupDelaySeconds +
         ", startupIntervalSeconds=" + startupIntervalSeconds +

SingularityBase/src/main/java/com/hubspot/deploy/HealthcheckOptionsBuilder.java

@@ -6,6 +6,7 @@
 import javax.validation.constraints.NotNull;
 
 import com.google.common.base.Optional;
+import com.hubspot.singularity.HealthcheckMethod;
 import com.hubspot.singularity.HealthcheckProtocol;
 
 public class HealthcheckOptionsBuilder {
@@ -14,6 +15,7 @@ public class HealthcheckOptionsBuilder {
   private Optional<Integer> portIndex;
   private Optional<Long> portNumber;
   private Optional<HealthcheckProtocol> protocol;
+  private Optional<HealthcheckMethod> method;
   private Optional<Integer> startupTimeoutSeconds;
   private Optional<Integer> startupDelaySeconds;
   private Optional<Integer> startupIntervalSeconds;
@@ -32,6 +34,7 @@ public HealthcheckOptionsBuilder(Optional<String> uri) {
     this.portIndex = Optional.absent();
     this.portNumber = Optional.absent();
     this.protocol = Optional.absent();
+    this.method = Optional.absent();
     this.startupTimeoutSeconds = Optional.absent();
     this.startupDelaySeconds = Optional.absent();
     this.startupIntervalSeconds = Optional.absent();
@@ -78,6 +81,15 @@ public HealthcheckOptionsBuilder setProtocol(Optional<HealthcheckProtocol> proto
     return this;
   }
 
+  public Optional<HealthcheckMethod> getMethod() {
+    return method;
+  }
+
+  public HealthcheckOptionsBuilder setMethod(Optional<HealthcheckMethod> method) {
+    this.method = method;
+    return this;
+  }
+
   public Optional<Integer> getStartupTimeoutSeconds() {
     return startupTimeoutSeconds;
   }
@@ -151,7 +163,7 @@ public HealthcheckOptionsBuilder setHealthcheckResultFilePath(Optional<String> h
   }
 
   public HealthcheckOptions build() {
-    return new HealthcheckOptions(uri, portIndex, portNumber, protocol, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
+    return new HealthcheckOptions(uri, portIndex, portNumber, protocol, method, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
   }
 
   @Override
@@ -167,6 +179,7 @@ public boolean equals(Object o) {
         Objects.equals(portIndex, that.portIndex) &&
         Objects.equals(portNumber, that.portNumber) &&
         Objects.equals(protocol, that.protocol) &&
+        Objects.equals(method, that.method) &&
         Objects.equals(startupTimeoutSeconds, that.startupTimeoutSeconds) &&
         Objects.equals(startupDelaySeconds, that.startupDelaySeconds) &&
         Objects.equals(startupIntervalSeconds, that.startupIntervalSeconds) &&
@@ -179,7 +192,7 @@ public boolean equals(Object o) {
 
   @Override
   public int hashCode() {
-    return Objects.hash(uri, portIndex, portNumber, protocol, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
+    return Objects.hash(uri, portIndex, portNumber, protocol, method, startupTimeoutSeconds, startupDelaySeconds, startupIntervalSeconds, intervalSeconds, responseTimeoutSeconds, maxRetries, failureStatusCodes, healthcheckResultFilePath);
   }
 
   @Override
@@ -189,6 +202,7 @@ public String toString() {
         ", portIndex=" + portIndex +
         ", portNumber=" + portNumber +
         ", protocol=" + protocol +
+        ", method=" + method +
         ", startupTimeoutSeconds=" + startupTimeoutSeconds +
         ", startupDelaySeconds=" + startupDelaySeconds +
         ", startupIntervalSeconds=" + startupIntervalSeconds +

SingularityBase/src/main/java/com/hubspot/singularity/HealthcheckMethod.java

@@ -0,0 +1,16 @@
+package com.hubspot.singularity;
+
+public enum  HealthcheckMethod {
+
+  GET("GET"), POST("POST");
+
+  private String method;
+
+  private HealthcheckMethod(String method) {
+    this.method = method;
+  }
+
+  public String getMethod() {
+    return method;
+  }
+}

SingularityBase/src/main/java/com/hubspot/singularity/HealthcheckProtocol.java

@@ -5,7 +5,7 @@
 @Schema
 public enum HealthcheckProtocol {
 
-  HTTP("http"), HTTPS("https");
+  HTTP("http"), HTTPS("https"), HTTP2("http"), HTTPS2("https");
 
   private final String protocol;
 

SingularityBase/src/main/java/com/hubspot/singularity/SingularityDeploy.java

@@ -203,6 +203,7 @@ public SingularityDeploy(@JsonProperty("requestId") String requestId,
         healthcheckPortIndex,
         Optional.<Long>absent(),
         healthcheckProtocol,
+        Optional.<HealthcheckMethod>absent(),
         Optional.<Integer>absent(),
         Optional.<Integer>absent(),
         Optional.<Integer>absent(),

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java

@@ -45,8 +45,8 @@ protected void configure() {
   @Named(LOCAL_DOWNLOAD_HTTP_CLIENT)
   public AsyncHttpClient providesHttpClient(SingularityExecutorConfiguration configuration) {
     AsyncHttpClientConfig.Builder configBldr = new AsyncHttpClientConfig.Builder();
-    configBldr.setRequestTimeoutInMs((int) configuration.getLocalDownloadServiceTimeoutMillis());
-    configBldr.setIdleConnectionTimeoutInMs((int) configuration.getLocalDownloadServiceTimeoutMillis());
+    configBldr.setRequestTimeout((int) configuration.getLocalDownloadServiceTimeoutMillis());
+    configBldr.setPooledConnectionIdleTimeout((int) configuration.getLocalDownloadServiceTimeoutMillis());
     configBldr.addRequestFilter(new ThrottleRequestFilter(configuration.getLocalDownloadServiceMaxConnections()));
 
     return new AsyncHttpClient(configBldr.build());

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java

@@ -1,6 +1,5 @@
 package com.hubspot.singularity.executor.task;
 
-import java.io.IOException;
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Objects;
@@ -172,8 +171,8 @@ private void downloadFilesFromLocalDownloadService(List<? extends S3Artifact> s3
           ListenableFuture<Response> future = localDownloadHttpClient.executeRequest(postRequestBldr.build());
 
           futures.add(new FutureHolder(future, System.currentTimeMillis(), s3Artifact));
-        } catch (IOException ioe) {
-          throw Throwables.propagate(ioe);
+        } catch (Throwable t) {
+          throw new RuntimeException(t);
         }
       }
 

SingularityService/pom.xml

@@ -14,6 +14,16 @@
     <basepom.shaded.main-class>com.hubspot.singularity.SingularityService</basepom.shaded.main-class>
   </properties>
 
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.squareup.okhttp3</groupId>
+        <artifactId>okhttp</artifactId>
+        <version>3.10.0</version>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
   <dependencies>
 
     <dependency>
@@ -136,6 +146,11 @@
       <artifactId>async-http-client</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.squareup.okhttp3</groupId>
+      <artifactId>okhttp</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>com.palominolabs.metrics</groupId>
       <artifactId>metrics-guice</artifactId>

SingularityService/src/main/java/com/hubspot/singularity/SingularityAuthModule.java

@@ -29,8 +29,8 @@ public void configure(Binder binder) {
       if (clazz == SingularityAuthenticatorClass.WEBHOOK) {
         AuthConfiguration authConfiguration = getConfiguration().getAuthConfiguration();
         AsyncHttpClientConfig clientConfig = new AsyncHttpClientConfig.Builder()
-            .setConnectionTimeoutInMs(authConfiguration.getWebhookAuthConnectTimeoutMs())
-            .setRequestTimeoutInMs(authConfiguration.getWebhookAuthRequestTimeoutMs())
+            .setConnectTimeout(authConfiguration.getWebhookAuthConnectTimeoutMs())
+            .setRequestTimeout(authConfiguration.getWebhookAuthRequestTimeoutMs())
             .setMaxRequestRetry(authConfiguration.getWebhookAuthRetries())
             .build();
         SingularityAsyncHttpClient webhookAsyncHttpClient = new SingularityAsyncHttpClient(clientConfig);

SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java

@@ -89,6 +89,7 @@
 import io.dropwizard.jetty.HttpsConnectorFactory;
 import io.dropwizard.server.DefaultServerFactory;
 import io.dropwizard.server.SimpleServerFactory;
+import okhttp3.OkHttpClient;
 
 
 public class SingularityMainModule implements Module {
@@ -158,6 +159,7 @@ public void configure(Binder binder) {
     binder.bind(MetricRegistry.class).toProvider(DropwizardMetricRegistryProvider.class).in(Scopes.SINGLETON);
 
     binder.bind(AsyncHttpClient.class).to(SingularityAsyncHttpClient.class).in(Scopes.SINGLETON);
+    binder.bind(OkHttpClient.class).to(SingularityOkHttpClient.class).in(Scopes.SINGLETON);
     binder.bind(ServerProvider.class).in(Scopes.SINGLETON);
 
     binder.bind(SingularityDropwizardHealthcheck.class).in(Scopes.SINGLETON);

SingularityService/src/main/java/com/hubspot/singularity/SingularityOkHttpClient.java

@@ -0,0 +1,34 @@
+package com.hubspot.singularity;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+
+import io.dropwizard.lifecycle.Managed;
+import okhttp3.OkHttpClient;
+
+public class SingularityOkHttpClient extends OkHttpClient implements Managed {
+  private static final Logger LOG = LoggerFactory.getLogger(SingularityOkHttpClient.class);
+
+  @Inject
+  SingularityOkHttpClient() {}
+
+  @Override
+  public void start() {}
+
+  @Override
+  public void stop() {
+    dispatcher().executorService().shutdown();
+    connectionPool().evictAll();
+    if (cache() != null) {
+      try {
+        cache().delete();
+      } catch (IOException e) {
+        LOG.warn("Unable to clean up client cache!");
+      }
+    }
+  }
+}

SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java

@@ -146,6 +146,8 @@ public class SingularityConfiguration extends Configuration {
 
   private long deleteDeadSlavesAfterHours = TimeUnit.DAYS.toHours(7);
 
+  private int maxMachineHistoryEntries = 10;
+
   private long deleteStaleRequestsFromZkWhenNoDatabaseAfterHours = TimeUnit.DAYS.toHours(14);
 
   private Optional<Integer> maxRequestsWithHistoryInZkWhenNoDatabase = Optional.absent();
@@ -251,6 +253,8 @@ public class SingularityConfiguration extends Configuration {
 
   private long reconcileSlavesEveryMinutes = TimeUnit.HOURS.toMinutes(1);
 
+  private long cleanInactiveHostListEveryHours = 24;
+
   @JsonProperty("s3")
   private S3Configuration s3Configuration;
 
@@ -688,6 +692,14 @@ public void setDeleteDeadSlavesAfterHours(long deleteDeadSlavesAfterHours) {
     this.deleteDeadSlavesAfterHours = deleteDeadSlavesAfterHours;
   }
 
+  public int getMaxMachineHistoryEntries() {
+    return maxMachineHistoryEntries;
+  }
+
+  public void setMaxMachineHistoryEntries(int maxMachineHistoryEntries) {
+    this.maxMachineHistoryEntries = maxMachineHistoryEntries;
+  }
+
   public int getListenerThreadpoolSize() {
     return listenerThreadpoolSize;
   }
@@ -1219,6 +1231,14 @@ public void setReconcileSlavesEveryMinutes(long reconcileSlavesEveryMinutes) {
     this.reconcileSlavesEveryMinutes = reconcileSlavesEveryMinutes;
   }
 
+  public long getCleanInactiveHostListEveryHours() {
+    return cleanInactiveHostListEveryHours;
+  }
+
+  public void setCleanInactiveHostListEveryHours(long cleanInactiveHostListEveryHours) {
+    this.cleanInactiveHostListEveryHours = cleanInactiveHostListEveryHours;
+  }
+
   public long getCacheTasksForMillis() {
     return cacheTasksForMillis;
   }