Skip to content

Commit

Permalink
Move S3 file configuration to SingularityService
Browse files Browse the repository at this point in the history
  • Loading branch information
ssalinas committed Jan 6, 2017
1 parent 80e9d0a commit 2ea0702
Show file tree
Hide file tree
Showing 23 changed files with 695 additions and 408 deletions.
13 changes: 2 additions & 11 deletions SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ public class ExecutorData {
private final Optional<Long> sigKillProcessesAfterMillis;
private final Optional<Integer> maxTaskThreads;
private final Optional<Boolean> preserveTaskSandboxAfterFinish;
private final Optional<String> loggingS3Bucket;
private final Optional<Integer> maxOpenFiles;
private final Optional<Boolean> skipLogrotateAndCompress;
private final Optional<List<S3ArtifactSignature>> s3ArtifactSignatures;
Expand All @@ -39,8 +38,7 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti
@JsonProperty("s3Artifacts") List<S3Artifact> s3Artifacts, @JsonProperty("successfulExitCodes") List<Integer> successfulExitCodes, @JsonProperty("user") Optional<String> user,
@JsonProperty("runningSentinel") Optional<String> runningSentinel, @JsonProperty("extraCmdLineArgs") List<String> extraCmdLineArgs, @JsonProperty("loggingTag") Optional<String> loggingTag,
@JsonProperty("loggingExtraFields") Map<String, String> loggingExtraFields, @JsonProperty("sigKillProcessesAfterMillis") Optional<Long> sigKillProcessesAfterMillis,
@JsonProperty("maxTaskThreads") Optional<Integer> maxTaskThreads, @JsonProperty("preserveTaskSandboxAfterFinish") Optional<Boolean> preserveTaskSandboxAfterFinish,
@JsonProperty("loggingS3Bucket") Optional<String> loggingS3Bucket, @JsonProperty("maxOpenFiles") Optional<Integer> maxOpenFiles,
@JsonProperty("maxTaskThreads") Optional<Integer> maxTaskThreads, @JsonProperty("preserveTaskSandboxAfterFinish") Optional<Boolean> preserveTaskSandboxAfterFinish, @JsonProperty("maxOpenFiles") Optional<Integer> maxOpenFiles,
@JsonProperty("skipLogrotateAndCompress") Optional<Boolean> skipLogrotateAndCompress, @JsonProperty("s3ArtifactSignatures") Optional<List<S3ArtifactSignature>> s3ArtifactSignatures,
@JsonProperty("logrotateFrequency") Optional<SingularityExecutorLogrotateFrequency> logrotateFrequency) {
this.cmd = cmd;
Expand All @@ -56,7 +54,6 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti
this.sigKillProcessesAfterMillis = sigKillProcessesAfterMillis;
this.maxTaskThreads = maxTaskThreads;
this.preserveTaskSandboxAfterFinish = preserveTaskSandboxAfterFinish;
this.loggingS3Bucket = loggingS3Bucket;
this.maxOpenFiles = maxOpenFiles;
this.skipLogrotateAndCompress = skipLogrotateAndCompress;
this.s3ArtifactSignatures = s3ArtifactSignatures;
Expand All @@ -65,7 +62,7 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti

public ExecutorDataBuilder toBuilder() {
return new ExecutorDataBuilder(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, runningSentinel, user, extraCmdLineArgs, loggingTag,
loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, loggingS3Bucket, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency);
loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency);
}

@ApiModelProperty(required=true, value="Command for the custom executor to run")
Expand Down Expand Up @@ -133,11 +130,6 @@ public Optional<Boolean> getPreserveTaskSandboxAfterFinish() {
return preserveTaskSandboxAfterFinish;
}

@ApiModelProperty(required=false, value="Override the default bucket used by the S3Uploader to store log files")
public Optional<String> getLoggingS3Bucket() {
return loggingS3Bucket;
}

@ApiModelProperty(required=false, value="Maximum number of open files the task process is allowed")
public Optional<Integer> getMaxOpenFiles() {
return maxOpenFiles;
Expand Down Expand Up @@ -174,7 +166,6 @@ public String toString() {
.add("sigKillProcessesAfterMillis", sigKillProcessesAfterMillis)
.add("maxTaskThreads", maxTaskThreads)
.add("preserveTaskSandboxAfterFinish", preserveTaskSandboxAfterFinish)
.add("loggingS3Bucket", loggingS3Bucket)
.add("maxOpenFiles", maxOpenFiles)
.add("skipLogrotateAndCompress", skipLogrotateAndCompress)
.add("s3ArtifactSignatures", s3ArtifactSignatures)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,14 @@ public class ExecutorDataBuilder {
private Optional<Long> sigKillProcessesAfterMillis;
private Optional<Integer> maxTaskThreads;
private Optional<Boolean> preserveTaskSandboxAfterFinish;
private Optional<String> loggingS3Bucket;
private Optional<Integer> maxOpenFiles;
private Optional<Boolean> skipLogrotateAndCompress;
private Optional<List<S3ArtifactSignature>> s3ArtifactSignatures;
private Optional<SingularityExecutorLogrotateFrequency> logrotateFrequency;

public ExecutorDataBuilder(String cmd, List<EmbeddedArtifact> embeddedArtifacts, List<ExternalArtifact> externalArtifacts, List<S3Artifact> s3Artifacts, List<Integer> successfulExitCodes,
Optional<String> runningSentinel, Optional<String> user, List<String> extraCmdLineArgs, Optional<String> loggingTag, Map<String, String> loggingExtraFields,
Optional<Long> sigKillProcessesAfterMillis, Optional<Integer> maxTaskThreads, Optional<Boolean> preserveTaskSandboxAfterFinish, Optional<String> loggingS3Bucket,
Optional<Long> sigKillProcessesAfterMillis, Optional<Integer> maxTaskThreads, Optional<Boolean> preserveTaskSandboxAfterFinish,
Optional<Integer> maxOpenFiles, Optional<Boolean> skipLogrotateAndCompress, Optional<List<S3ArtifactSignature>> s3ArtifactSignatures, Optional<SingularityExecutorLogrotateFrequency> logrotateFrequency) {
this.cmd = cmd;
this.embeddedArtifacts = embeddedArtifacts;
Expand All @@ -44,7 +43,6 @@ public ExecutorDataBuilder(String cmd, List<EmbeddedArtifact> embeddedArtifacts,
this.sigKillProcessesAfterMillis = sigKillProcessesAfterMillis;
this.maxTaskThreads = maxTaskThreads;
this.preserveTaskSandboxAfterFinish = preserveTaskSandboxAfterFinish;
this.loggingS3Bucket = loggingS3Bucket;
this.maxOpenFiles = maxOpenFiles;
this.skipLogrotateAndCompress = skipLogrotateAndCompress;
this.s3ArtifactSignatures = s3ArtifactSignatures;
Expand All @@ -57,7 +55,7 @@ public ExecutorDataBuilder() {

public ExecutorData build() {
return new ExecutorData(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, user, runningSentinel, extraCmdLineArgs, loggingTag, loggingExtraFields,
sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, loggingS3Bucket, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency);
sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency);
}

public Optional<String> getLoggingTag() {
Expand Down Expand Up @@ -177,15 +175,6 @@ public ExecutorDataBuilder setPreserveTaskSandboxAfterFinish(Optional<Boolean> p
return this;
}

public Optional<String> getLoggingS3Bucket() {
return loggingS3Bucket;
}

public ExecutorDataBuilder setLoggingS3Bucket(Optional<String> loggingS3Bucket) {
this.loggingS3Bucket = loggingS3Bucket;
return this;
}

public Optional<Integer> getMaxOpenFiles() {
return maxOpenFiles;
}
Expand Down Expand Up @@ -238,7 +227,6 @@ public String toString() {
", sigKillProcessesAfterMillis=" + sigKillProcessesAfterMillis +
", maxTaskThreads=" + maxTaskThreads +
", preserveTaskSandboxAfterFinish=" + preserveTaskSandboxAfterFinish +
", loggingS3Bucket=" + loggingS3Bucket +
", maxOpenFiles=" + maxOpenFiles +
", skipLogrotateAndCompress=" + skipLogrotateAndCompress +
", s3ArtifactSignatures=" + s3ArtifactSignatures +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,28 +13,30 @@
import com.google.common.collect.Sets;

public class SingularityS3FormatHelper {
public static final String DEFAULT_GROUP_NAME = "default";

private static final List<String> DISALLOWED_FOR_TASK = ImmutableList.of("%index", "%s", "%filename", "%fileext");
private static final List<String> DISALLOWED_FOR_DEPLOY = ImmutableList.copyOf(Iterables.concat(DISALLOWED_FOR_TASK, ImmutableList.of("%host")));
private static final List<String> DISALLOWED_FOR_REQUEST = ImmutableList.copyOf(Iterables.concat(DISALLOWED_FOR_DEPLOY, ImmutableList.of("%tag", "%deployId")));

public static String getS3KeyFormat(String s3KeyFormat, String requestId) {
public static String getS3KeyFormat(String s3KeyFormat, String requestId, String group) {
s3KeyFormat = s3KeyFormat.replace("%requestId", requestId);
s3KeyFormat = s3KeyFormat.replace("%group", group);

return s3KeyFormat;
}

public static String getS3KeyFormat(String s3KeyFormat, String requestId, String deployId, Optional<String> loggingTag) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId);
public static String getS3KeyFormat(String s3KeyFormat, String requestId, String deployId, Optional<String> loggingTag, String group) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group);

s3KeyFormat = s3KeyFormat.replace("%tag", loggingTag.or(""));
s3KeyFormat = s3KeyFormat.replace("%deployId", deployId);

return s3KeyFormat;
}

public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional<String> loggingTag) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag);
public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional<String> loggingTag, String group) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag, group);

s3KeyFormat = s3KeyFormat.replace("%host", taskId.getSanitizedHost());
s3KeyFormat = s3KeyFormat.replace("%taskId", taskId.toString());
Expand Down Expand Up @@ -118,8 +120,8 @@ private static String getDayOrMonth(int value) {
return String.format("%02d", value);
}

public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, String deployId, Optional<String> tag, long start, long end) {
String keyFormat = getS3KeyFormat(s3KeyFormat, requestId, deployId, tag);
public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, String deployId, Optional<String> tag, long start, long end, String group) {
String keyFormat = getS3KeyFormat(s3KeyFormat, requestId, deployId, tag, group);

keyFormat = trimTaskId(keyFormat, requestId + "-" + deployId);

Expand All @@ -136,8 +138,8 @@ private static String trimTaskId(String s3KeyFormat, String replaceWith) {
return s3KeyFormat;
}

public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, long start, long end) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId);
public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, String requestId, long start, long end, String group) {
s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group);

s3KeyFormat = trimTaskId(s3KeyFormat, requestId);

Expand Down Expand Up @@ -206,8 +208,8 @@ private static Collection<String> getS3KeyPrefixes(String s3KeyFormat, List<Stri
return keyPrefixes;
}

public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, SingularityTaskId taskId, Optional<String> tag, long start, long end) {
String keyFormat = getS3KeyFormat(s3KeyFormat, taskId, tag);
public static Collection<String> getS3KeyPrefixes(String s3KeyFormat, SingularityTaskId taskId, Optional<String> tag, long start, long end, String group) {
String keyFormat = getS3KeyFormat(s3KeyFormat, taskId, tag, group);

return getS3KeyPrefixes(keyFormat, DISALLOWED_FOR_TASK, start, end);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,75 +7,33 @@
import com.wordnik.swagger.annotations.ApiModelProperty;

@ApiModel( description = "Represents a task sandbox file that was uploaded to S3" )
public class SingularityS3Log {
public static final String LOG_START_S3_ATTR = "starttime";
public static final String LOG_END_S3_ATTR = "endtime";

public class SingularityS3Log extends SingularityS3LogMetadata {
private final String getUrl;
private final String key;
private final long lastModified;
private final long size;
private final String downloadUrl;
private final Optional<Long> startTime;
private final Optional<Long> endTime;

@JsonCreator
public SingularityS3Log(@JsonProperty("getUrl") String getUrl, @JsonProperty("key") String key, @JsonProperty("lastModified") long lastModified, @JsonProperty("size") long size, @JsonProperty("downloadUrl") String downloadUrl,
@JsonProperty("startTime") Optional<Long> startTime, @JsonProperty("endTime") Optional<Long> endTime) {
super(key, lastModified, size, startTime, endTime);
this.getUrl = getUrl;
this.key = key;
this.lastModified = lastModified;
this.size = size;
this.downloadUrl = downloadUrl;
this.startTime = startTime;
this.endTime = endTime;
}

@ApiModelProperty("URL to file in S3")
public String getGetUrl() {
return getUrl;
}

@ApiModelProperty("S3 key")
public String getKey() {
return key;
}

@ApiModelProperty("Last modified time")
public long getLastModified() {
return lastModified;
}

@ApiModelProperty("File size (in bytes)")
public long getSize() {
return size;
}

@ApiModelProperty("URL to file in S3 containing headers that will force file to be downloaded instead of viewed")
public String getDownloadUrl() {
return downloadUrl;
}

@ApiModelProperty("Time the log file started being written to")
public Optional<Long> getStartTime() {
return startTime;
}

@ApiModelProperty("Time the log file was finished being written to")
public Optional<Long> getEndTime() {
return endTime;
}

@Override
public String toString() {
return "SingularityS3Log{" +
"getUrl='" + getUrl + '\'' +
", key='" + key + '\'' +
", lastModified=" + lastModified +
", size=" + size +
", downloadUrl='" + downloadUrl + '\'' +
", startTime=" + startTime +
", endTime=" + endTime +
'}';
"} " + super.toString();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package com.hubspot.singularity;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Optional;
import com.wordnik.swagger.annotations.ApiModelProperty;

public class SingularityS3LogMetadata {
public static final String LOG_START_S3_ATTR = "starttime";
public static final String LOG_END_S3_ATTR = "endtime";

private final String key;
private final long lastModified;
private final long size;
private final Optional<Long> startTime;
private final Optional<Long> endTime;

@JsonCreator
public SingularityS3LogMetadata(@JsonProperty("key") String key, @JsonProperty("lastModified") long lastModified, @JsonProperty("size") long size,
@JsonProperty("startTime") Optional<Long> startTime, @JsonProperty("endTime") Optional<Long> endTime) {
this.key = key;
this.lastModified = lastModified;
this.size = size;
this.startTime = startTime;
this.endTime = endTime;
}

@ApiModelProperty("S3 key")
public String getKey() {
return key;
}

@ApiModelProperty("Last modified time")
public long getLastModified() {
return lastModified;
}

@ApiModelProperty("File size (in bytes)")
public long getSize() {
return size;
}

@ApiModelProperty("Time the log file started being written to")
public Optional<Long> getStartTime() {
return startTime;
}

@ApiModelProperty("Time the log file was finished being written to")
public Optional<Long> getEndTime() {
return endTime;
}

@Override
public String toString() {
return "SingularityS3Log{" +
"key='" + key + '\'' +
", lastModified=" + lastModified +
", size=" + size +
", startTime=" + startTime +
", endTime=" + endTime +
'}';
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package com.hubspot.singularity;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Optional;

public class SingularityS3UploaderFile {
private final String filename;
private final Optional<String> s3UploaderBucket;
private final Optional<String> s3UploaderKeyPattern;
private final Optional<String> s3UploaderFilenameHint;
private final Optional<String> directory;

@JsonCreator
public static SingularityS3UploaderFile fromString(String value) {
return new SingularityS3UploaderFile(value, Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent(), Optional.<String>absent());
}

@JsonCreator
public SingularityS3UploaderFile(@JsonProperty("filename") String filename,
@JsonProperty("s3UploaderBucket") Optional<String> s3UploaderBucket,
@JsonProperty("s3UploaderKeyPattern") Optional<String> s3UploaderKeyPattern,
@JsonProperty("s3UploaderFilenameHint") Optional<String> s3UploaderFilenameHint,
@JsonProperty("directory") Optional<String> directory) {
this.filename = filename;
this.s3UploaderBucket = s3UploaderBucket;
this.s3UploaderKeyPattern = s3UploaderKeyPattern;
this.s3UploaderFilenameHint = s3UploaderFilenameHint;
this.directory = directory;
}

public String getFilename() {
return filename;
}

public Optional<String> getS3UploaderBucket() {
return s3UploaderBucket;
}

public Optional<String> getS3UploaderKeyPattern() {
return s3UploaderKeyPattern;
}

public Optional<String> getS3UploaderFilenameHint() {
return s3UploaderFilenameHint;
}

public Optional<String> getDirectory() {
return directory;
}

@Override
public String toString() {
return "SingularityS3UploaderFile{" +
"filename='" + filename + '\'' +
", s3UploaderBucket=" + s3UploaderBucket +
", s3UploaderKeyPattern=" + s3UploaderKeyPattern +
", s3UploaderFilenameHint=" + s3UploaderFilenameHint +
", directory=" + directory +
'}';
}
}
Loading

0 comments on commit 2ea0702

Please sign in to comment.