Skip to content

Commit

Permalink
allow each additional file to specify s3 storage class
Browse files Browse the repository at this point in the history
  • Loading branch information
@wsorenson
wsorenson committed Dec 14, 2016
1 parent 57f3ffe commit e26e8dc
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 59 deletions.
122 changes: 68 additions & 54 deletions .../com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,58 +5,72 @@
import com.google.common.base.Optional;

public class SingularityExecutorS3UploaderAdditionalFile {
private final String filename;
private final Optional<String> s3UploaderBucket;
private final Optional<String> s3UploaderKeyPattern;
private final Optional<String> s3UploaderFilenameHint;
private final Optional<String> directory;

@JsonCreator
public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) {
return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
}

@JsonCreator
public SingularityExecutorS3UploaderAdditionalFile(@JsonProperty("filename") String filename,
@JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
@JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
@JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
@JsonProperty("directory") Optional<String> directory) {
this.filename = filename;
this.s3UploaderBucket = s3UploaderBucket;
this.s3UploaderKeyPattern = s3UploaderKeyPattern;
this.s3UploaderFilenameHint = s3UploaderFilenameHint;
this.directory = directory;
}

public String getFilename() {
return filename;
}

public Optional<String> getS3UploaderBucket() {
return s3UploaderBucket;
}

public Optional<String> getS3UploaderKeyPattern() {
return s3UploaderKeyPattern;
}

public Optional<String> getS3UploaderFilenameHint() {
return s3UploaderFilenameHint;
}

public Optional<String> getDirectory() {
return directory;
}

@Override
public String toString() {
return "SingularityExecutorS3UploaderAdditionalFile[" +
"filename='" + filename + '\'' +
", s3UploaderBucket=" + s3UploaderBucket +
", s3UploaderKeyPattern=" + s3UploaderKeyPattern +
", s3UploaderFilenameHint=" + s3UploaderFilenameHint +
", directory=" + directory +
']';
}

private final String filename;
private final Optional<String> s3UploaderBucket;
private final Optional<String> s3UploaderKeyPattern;
private final Optional<String> s3UploaderFilenameHint;
private final Optional<String> directory;
private final Optional<String> s3StorageClass;
private final Optional<Long> applyS3StorageClassAfterBytes;

@JsonCreator
public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) {
return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(),
Optional.<Long> absent());
}

@JsonCreator
public SingularityExecutorS3UploaderAdditionalFile(
@JsonProperty("filename") String filename,
@JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
@JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
@JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
@JsonProperty("directory") Optional<String> directory,
@JsonProperty("s3StorageClass") Optional<String> s3StorageClass,
@JsonProperty("applyS3StorageClassAfterBytes") Optional<Long> applyS3StorageClassAfterBytes) {
this.filename = filename;
this.s3UploaderBucket = s3UploaderBucket;
this.s3UploaderKeyPattern = s3UploaderKeyPattern;
this.s3UploaderFilenameHint = s3UploaderFilenameHint;
this.directory = directory;
this.s3StorageClass = s3StorageClass;
this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes;
}

public String getFilename() {
return filename;
}

public Optional<String> getS3UploaderBucket() {
return s3UploaderBucket;
}

public Optional<String> getS3UploaderKeyPattern() {
return s3UploaderKeyPattern;
}

public Optional<String> getS3UploaderFilenameHint() {
return s3UploaderFilenameHint;
}

public Optional<String> getDirectory() {
return directory;
}

public Optional<String> getS3StorageClass() {
return s3StorageClass;
}

public Optional<Long> getApplyS3StorageClassAfterBytes() {
return applyS3StorageClassAfterBytes;
}

@Override
public String toString() {
return "SingularityExecutorS3UploaderAdditionalFile [filename=" + filename + ", s3UploaderBucket=" + s3UploaderBucket + ", s3UploaderKeyPattern=" + s3UploaderKeyPattern
+ ", s3UploaderFilenameHint=" + s3UploaderFilenameHint + ", directory=" + directory + ", s3StorageClass=" + s3StorageClass + ", applyS3StorageClassAfterBytes=" + applyS3StorageClassAfterBytes
+ "]";
}

}
15 changes: 10 additions & 5 deletions ...rc/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,13 +60,15 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) {
final Path serviceLogParent = serviceLogOutPath.getParent();
final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory());

boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(), Optional.<String>absent(), finished);
boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.<String>absent(),
Optional.<String>absent(), finished, configuration.getS3StorageClass(), configuration.getApplyS3StorageClassAfterBytes());

int index = 1;
for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) {
Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath();
String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.[gb]z*", additionalFile.getFilename());
result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished);
result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(),
additionalFile.getS3UploaderKeyPattern(), finished, additionalFile.getS3StorageClass(), additionalFile.getApplyS3StorageClassAfterBytes());
index++;
}

Expand Down Expand Up @@ -96,7 +98,9 @@ public boolean teardown() {
boolean writeS3MetadataForNonLogRotatedFileSuccess = true;

if (!taskDefinition.shouldLogrotateLogFile()) {
writeS3MetadataForNonLogRotatedFileSuccess = writeS3MetadataFile("unrotated", taskDefinition.getServiceLogOutPath().getParent(), taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.<String>absent(), Optional.<String>absent(), true);
writeS3MetadataForNonLogRotatedFileSuccess = writeS3MetadataFile("unrotated", taskDefinition.getServiceLogOutPath().getParent(),
taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.<String>absent(), Optional.<String>absent(), true, configuration.getS3StorageClass(),
configuration.getApplyS3StorageClassAfterBytes());
}

if (manualLogrotate()) {
Expand Down Expand Up @@ -229,13 +233,14 @@ public Path getLogrotateCronPath() {
return Paths.get(configuration.getCronDirectory()).resolve(taskDefinition.getTaskId() + ".logrotate");
}

private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional<String> s3Bucket, Optional<String> s3KeyPattern, boolean finished) {
private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional<String> s3Bucket, Optional<String> s3KeyPattern, boolean finished,
Optional<String> s3StorageClass, Optional<Long> applyS3StorageClassAfterBytes) {
final String s3UploaderBucket = taskDefinition.getExecutorData().getLoggingS3Bucket().or(configuration.getS3UploaderBucket());

S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3Bucket.or(s3UploaderBucket),
getS3KeyPattern(s3KeyPattern.or(configuration.getS3UploaderKeyPattern())), finished, Optional.<String> absent(),
Optional.<Integer> absent(), Optional.<String> absent(), Optional.<String> absent(), Optional.<Long> absent(),
configuration.getS3StorageClass(), configuration.getApplyS3StorageClassAfterBytes());
s3StorageClass, applyS3StorageClassAfterBytes);

String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix());

Expand Down

1 comment on commit e26e8dc

@tpetr
Copy link
Contributor

@tpetr tpetr commented on e26e8dc Jan 19, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🚢

Please sign in to comment.