Merge pull request #1373 from HubSpot/custom-use-compress-program

customizable --use-compress-program for tar
HubSpot · Jan 11, 2017 · edb97a9 · edb97a9
2 parents 48b269d + a5998b5
commit edb97a9
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 12 deletions.
diff --git a/...c/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java b/...c/main/java/com/hubspot/singularity/executor/task/SingularityExecutorArtifactFetcher.java
@@ -22,6 +22,7 @@
 import com.hubspot.mesos.JavaUtils;
 import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration;
 import com.hubspot.singularity.executor.config.SingularityExecutorModule;
+import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration;
 import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier;
 import com.hubspot.singularity.s3.base.ArtifactDownloadRequest;
 import com.hubspot.singularity.s3.base.ArtifactManager;
@@ -41,21 +42,23 @@ public class SingularityExecutorArtifactFetcher {
   private final SingularityS3Configuration s3Configuration;
   private final ObjectMapper objectMapper;
   private final SingularityRunnerExceptionNotifier exceptionNotifier;
+  private final SingularityRunnerBaseConfiguration runnerBaseConfiguration;
 
   @Inject
   public SingularityExecutorArtifactFetcher(@Named(SingularityExecutorModule.LOCAL_DOWNLOAD_HTTP_CLIENT) AsyncHttpClient localDownloadHttpClient, SingularityS3Configuration s3Configuration,
-      SingularityExecutorConfiguration executorConfiguration, ObjectMapper objectMapper, SingularityRunnerExceptionNotifier exceptionNotifier) {
+      SingularityExecutorConfiguration executorConfiguration, ObjectMapper objectMapper, SingularityRunnerExceptionNotifier exceptionNotifier, SingularityRunnerBaseConfiguration runnerBaseConfiguration) {
     this.localDownloadHttpClient = localDownloadHttpClient;
     this.executorConfiguration = executorConfiguration;
     this.s3Configuration = s3Configuration;
     this.objectMapper = objectMapper;
     this.exceptionNotifier = exceptionNotifier;
+    this.runnerBaseConfiguration = runnerBaseConfiguration;
 
     this.localDownloadUri = String.format(LOCAL_DOWNLOAD_STRING_FORMAT, s3Configuration.getLocalDownloadHttpPort(), s3Configuration.getLocalDownloadPath());
   }
 
   public SingularityExecutorTaskArtifactFetcher buildTaskFetcher(ExecutorData executorData, SingularityExecutorTask task) {
-    ArtifactManager artifactManager = new ArtifactManager(s3Configuration, task.getLog(), exceptionNotifier);
+    ArtifactManager artifactManager = new ArtifactManager(runnerBaseConfiguration, s3Configuration, task.getLog(), exceptionNotifier);
 
     return new SingularityExecutorTaskArtifactFetcher(artifactManager, executorData, task);
   }

diff --git a/...com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java b/...com/hubspot/singularity/runner/base/configuration/SingularityRunnerBaseConfiguration.java
@@ -34,6 +34,10 @@ public class SingularityRunnerBaseConfiguration extends BaseRunnerConfiguration
   @JsonProperty
   private String sentryPrefix = "";
 
+  @NotNull
+  @JsonProperty
+  private Optional<String> useCompressProgram = Optional.absent();
+
   public SingularityRunnerBaseConfiguration() {
     super(Optional.<String>absent());
     this.setLoggingDirectory(Optional.of("/var/log/singularity/"));
@@ -87,13 +91,22 @@ public void setSentryPrefix(String sentryPrefix) {
     this.sentryPrefix = sentryPrefix;
   }
 
+  public Optional<String> getUseCompressProgram() {
+    return useCompressProgram;
+  }
+
+  public void setUseCompressProgram(Optional<String> useCompressProgram) {
+    this.useCompressProgram = useCompressProgram;
+  }
+
   @Override
   public String toString() {
     return "SingularityRunnerBaseConfiguration[" +
             "s3UploaderMetadataDirectory='" + s3UploaderMetadataDirectory + '\'' +
             ", s3UploaderMetadataSuffix='" + s3UploaderMetadataSuffix + '\'' +
             ", logWatcherMetadataDirectory='" + logWatcherMetadataDirectory + '\'' +
             ", logWatcherMetadataSuffix='" + logWatcherMetadataSuffix + '\'' +
+            ", useCompressProgram=" + useCompressProgram +
             ']';
   }
 }
diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/ArtifactManager.java
@@ -13,6 +13,7 @@
 
 import org.slf4j.Logger;
 
+import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.hash.HashCode;
@@ -22,6 +23,7 @@
 import com.hubspot.deploy.ExternalArtifact;
 import com.hubspot.deploy.RemoteArtifact;
 import com.hubspot.deploy.S3Artifact;
+import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration;
 import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier;
 import com.hubspot.singularity.runner.base.shared.ProcessFailedException;
 import com.hubspot.singularity.runner.base.shared.SimpleProcessManager;
@@ -32,13 +34,15 @@ public class ArtifactManager extends SimpleProcessManager {
   private final Path cacheDirectory;
   private final Logger log;
   private final S3ArtifactDownloader s3ArtifactDownloader;
+  private final Optional<String> useCompressProgram;
 
-  public ArtifactManager(SingularityS3Configuration configuration, Logger log, SingularityRunnerExceptionNotifier exceptionNotifier) {
+  public ArtifactManager(SingularityRunnerBaseConfiguration runnerBaseConfiguration, SingularityS3Configuration configuration, Logger log, SingularityRunnerExceptionNotifier exceptionNotifier) {
     super(log);
 
     this.cacheDirectory = Paths.get(configuration.getArtifactCacheDirectory());
     this.log = log;
     this.s3ArtifactDownloader = new S3ArtifactDownloader(configuration, log, exceptionNotifier);
+    this.useCompressProgram = runnerBaseConfiguration.getUseCompressProgram();
   }
 
   private long getSize(Path path) {
@@ -207,14 +211,15 @@ public void copy(Path source, Path destination, String destinationFilename) {
   public void untar(Path source, Path destination) {
     log.info("Untarring {} to {}", source, destination);
 
-    final List<String> command = ImmutableList.of(
-        "tar",
-        "-oxzf",
-        source.toString(),
-        "-C",
-        destination.toString());
+    final ImmutableList.Builder<String> commandBuilder = ImmutableList.<String>builder().add("tar", "-oxf", source.toString(), "-C", destination.toString());
 
-    runCommandAndThrowRuntimeException(command);
+    if (useCompressProgram.isPresent()) {
+      commandBuilder.add("--use-compress-program=" + useCompressProgram.get());
+    } else {
+      commandBuilder.add("-z");
+    }
+
+    runCommandAndThrowRuntimeException(commandBuilder.build());
   }
 
   private void runCommandAndThrowRuntimeException(List<String> command) {

diff --git a/...er/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java b/...er/src/main/java/com/hubspot/singularity/s3downloader/config/ArtifactManagerProvider.java
@@ -5,26 +5,29 @@
 
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration;
 import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier;
 import com.hubspot.singularity.s3.base.ArtifactManager;
 import com.hubspot.singularity.s3.base.config.SingularityS3Configuration;
 
 public class ArtifactManagerProvider implements Provider<ArtifactManager> {
 
   private final Logger log;
+  private final SingularityRunnerBaseConfiguration runnerBaseConfiguration;
   private final SingularityS3Configuration s3Configuration;
   private final SingularityRunnerExceptionNotifier exceptionNotifier;
 
   @Inject
-  public ArtifactManagerProvider(SingularityS3Configuration s3Configuration, SingularityRunnerExceptionNotifier exceptionNotifier) {
+  public ArtifactManagerProvider(SingularityRunnerBaseConfiguration runnerBaseConfiguration, SingularityS3Configuration s3Configuration, SingularityRunnerExceptionNotifier exceptionNotifier) {
     this.log = LoggerFactory.getLogger(ArtifactManager.class);
+    this.runnerBaseConfiguration = runnerBaseConfiguration;
     this.s3Configuration = s3Configuration;
     this.exceptionNotifier = exceptionNotifier;
   }
 
   @Override
   public ArtifactManager get() {
-    return new ArtifactManager(s3Configuration, log, exceptionNotifier);
+    return new ArtifactManager(runnerBaseConfiguration, s3Configuration, log, exceptionNotifier);
   }
 
 }