diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java index c4e4ffb4cd..4fce096b0b 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java @@ -5,58 +5,72 @@ import com.google.common.base.Optional; public class SingularityExecutorS3UploaderAdditionalFile { - private final String filename; - private final Optional s3UploaderBucket; - private final Optional s3UploaderKeyPattern; - private final Optional s3UploaderFilenameHint; - private final Optional directory; - - @JsonCreator - public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) { - return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); - } - - @JsonCreator - public SingularityExecutorS3UploaderAdditionalFile(@JsonProperty("filename") String filename, - @JsonProperty("s3UploaderBucket") Optional s3UploaderBucket, - @JsonProperty("s3UploaderKeyPattern") Optional s3UploaderKeyPattern, - @JsonProperty("s3UploaderFilenameHint") Optional s3UploaderFilenameHint, - @JsonProperty("directory") Optional directory) { - this.filename = filename; - this.s3UploaderBucket = s3UploaderBucket; - this.s3UploaderKeyPattern = s3UploaderKeyPattern; - this.s3UploaderFilenameHint = s3UploaderFilenameHint; - this.directory = directory; - } - - public String getFilename() { - return filename; - } - - public Optional getS3UploaderBucket() { - return s3UploaderBucket; - } - - public Optional getS3UploaderKeyPattern() { - return s3UploaderKeyPattern; - } - - public Optional getS3UploaderFilenameHint() { - return s3UploaderFilenameHint; - } - - public Optional getDirectory() { - return directory; - } - - @Override - public String toString() { - return "SingularityExecutorS3UploaderAdditionalFile[" + - "filename='" + filename + '\'' + - ", s3UploaderBucket=" + s3UploaderBucket + - ", s3UploaderKeyPattern=" + s3UploaderKeyPattern + - ", s3UploaderFilenameHint=" + s3UploaderFilenameHint + - ", directory=" + directory + - ']'; - } + + private final String filename; + private final Optional s3UploaderBucket; + private final Optional s3UploaderKeyPattern; + private final Optional s3UploaderFilenameHint; + private final Optional directory; + private final Optional s3StorageClass; + private final Optional applyS3StorageClassAfterBytes; + + @JsonCreator + public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) { + return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), + Optional. absent()); + } + + @JsonCreator + public SingularityExecutorS3UploaderAdditionalFile( + @JsonProperty("filename") String filename, + @JsonProperty("s3UploaderBucket") Optional s3UploaderBucket, + @JsonProperty("s3UploaderKeyPattern") Optional s3UploaderKeyPattern, + @JsonProperty("s3UploaderFilenameHint") Optional s3UploaderFilenameHint, + @JsonProperty("directory") Optional directory, + @JsonProperty("s3StorageClass") Optional s3StorageClass, + @JsonProperty("applyS3StorageClassAfterBytes") Optional applyS3StorageClassAfterBytes) { + this.filename = filename; + this.s3UploaderBucket = s3UploaderBucket; + this.s3UploaderKeyPattern = s3UploaderKeyPattern; + this.s3UploaderFilenameHint = s3UploaderFilenameHint; + this.directory = directory; + this.s3StorageClass = s3StorageClass; + this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; + } + + public String getFilename() { + return filename; + } + + public Optional getS3UploaderBucket() { + return s3UploaderBucket; + } + + public Optional getS3UploaderKeyPattern() { + return s3UploaderKeyPattern; + } + + public Optional getS3UploaderFilenameHint() { + return s3UploaderFilenameHint; + } + + public Optional getDirectory() { + return directory; + } + + public Optional getS3StorageClass() { + return s3StorageClass; + } + + public Optional getApplyS3StorageClassAfterBytes() { + return applyS3StorageClassAfterBytes; + } + + @Override + public String toString() { + return "SingularityExecutorS3UploaderAdditionalFile [filename=" + filename + ", s3UploaderBucket=" + s3UploaderBucket + ", s3UploaderKeyPattern=" + s3UploaderKeyPattern + + ", s3UploaderFilenameHint=" + s3UploaderFilenameHint + ", directory=" + directory + ", s3StorageClass=" + s3StorageClass + ", applyS3StorageClassAfterBytes=" + applyS3StorageClassAfterBytes + + "]"; + } + } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index 10f8135249..e39b9a8b28 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -60,13 +60,15 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) { final Path serviceLogParent = serviceLogOutPath.getParent(); final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory()); - boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.absent(), Optional.absent(), finished); + boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.absent(), + Optional.absent(), finished, configuration.getS3StorageClass(), configuration.getApplyS3StorageClassAfterBytes()); int index = 1; for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) { Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath(); String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.[gb]z*", additionalFile.getFilename()); - result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished); + result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), + additionalFile.getS3UploaderKeyPattern(), finished, additionalFile.getS3StorageClass(), additionalFile.getApplyS3StorageClassAfterBytes()); index++; } @@ -96,7 +98,9 @@ public boolean teardown() { boolean writeS3MetadataForNonLogRotatedFileSuccess = true; if (!taskDefinition.shouldLogrotateLogFile()) { - writeS3MetadataForNonLogRotatedFileSuccess = writeS3MetadataFile("unrotated", taskDefinition.getServiceLogOutPath().getParent(), taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.absent(), Optional.absent(), true); + writeS3MetadataForNonLogRotatedFileSuccess = writeS3MetadataFile("unrotated", taskDefinition.getServiceLogOutPath().getParent(), + taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.absent(), Optional.absent(), true, configuration.getS3StorageClass(), + configuration.getApplyS3StorageClassAfterBytes()); } if (manualLogrotate()) { @@ -229,13 +233,14 @@ public Path getLogrotateCronPath() { return Paths.get(configuration.getCronDirectory()).resolve(taskDefinition.getTaskId() + ".logrotate"); } - private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional s3Bucket, Optional s3KeyPattern, boolean finished) { + private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional s3Bucket, Optional s3KeyPattern, boolean finished, + Optional s3StorageClass, Optional applyS3StorageClassAfterBytes) { final String s3UploaderBucket = taskDefinition.getExecutorData().getLoggingS3Bucket().or(configuration.getS3UploaderBucket()); S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3Bucket.or(s3UploaderBucket), getS3KeyPattern(s3KeyPattern.or(configuration.getS3UploaderKeyPattern())), finished, Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), - configuration.getS3StorageClass(), configuration.getApplyS3StorageClassAfterBytes()); + s3StorageClass, applyS3StorageClassAfterBytes); String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix());