diff --git a/Docs/releases/0.14.0.md b/Docs/releases/0.14.0.md new file mode 100644 index 0000000000..fe327efb3d --- /dev/null +++ b/Docs/releases/0.14.0.md @@ -0,0 +1,47 @@ +## Changes in `0.14.0` + +Check out the [0.14.0 milestone](https://github.com/HubSpot/Singularity/issues?q=milestone%3A0.14.0+is%3Aclosed) to see new features / bugfixes in detail. + +## Configuration Changes + +[#1391](https://github.com/HubSpot/Singularity/pull/1391) include a rework of some of the S3 settings in Singularity. If you use the `SingularityExecutor` or `SingularityExecutorCleanup` modules and use the S3 upload features, you will need an update to your configuration. The fields for specifying which files to upload have been moved out of the `SingularityExecutor`. An example below shows all fields that would move. + +Old Configuration (gets removed from `SingularityExecutor` and `SingularityExecutorCleanup` yaml files) +```yaml +executor: + s3UploaderBucket: my-logs-bucket + s3UploaderKeyPattern: "%requestId/%Y/%m/%taskId_%index-%s-%filename" + s3UploaderAdditionalFiles: + - access.log + s3StorageClass: "STANDARD_IA" + applyS3StorageClassAfterBytes: 75000 +``` + +New Configuration (if not already present for use with S3 log searching) +```yaml +# in SingularityExecutorCleanup yaml configuration +executorCleanup: + defaultS3Bucket: my-logs-bucket + s3KeyFormat: "%requestId/%Y/%m/%taskId_%index-%s-%filename" + s3StorageClass: "STANDARD_IA" + applyS3StorageClassAfterBytes: 75000 + s3UploaderAdditionalFiles: + - filename: access.log + # The default directory in the executor was set to 'logs', now it must be manually specified + # If not specified, the directory to search for log files will be the task app directory in the sandbox + directory: logs + +# in SingularityService yaml configuration +s3: + s3Bucket: my-logs-bucket + s3KeyFormat: "%requestId/%Y/%m/%taskId_%index-%s-%filename" + s3StorageClass: "STANDARD_IA" + applyS3StorageClassAfterBytes: 75000 + s3UploaderAdditionalFiles: + - filename: access.log + # The default directory in the executor was set to 'logs', now it must be manually specified + # If not specified, the directory to search for log files will be the task app directory in the sandbox + directory: logs +``` + +**NOTE** - To upgrade smoothly, it is strongly recommended to deploy `SingularityService` and the `SingularityExecutorCleanup` *before* deploying the `SingularityExecutor` diff --git a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java index 2a343125b4..54083a2bd0 100644 --- a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java +++ b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorData.java @@ -28,7 +28,6 @@ public class ExecutorData { private final Optional sigKillProcessesAfterMillis; private final Optional maxTaskThreads; private final Optional preserveTaskSandboxAfterFinish; - private final Optional loggingS3Bucket; private final Optional maxOpenFiles; private final Optional skipLogrotateAndCompress; private final Optional> s3ArtifactSignatures; @@ -39,8 +38,7 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti @JsonProperty("s3Artifacts") List s3Artifacts, @JsonProperty("successfulExitCodes") List successfulExitCodes, @JsonProperty("user") Optional user, @JsonProperty("runningSentinel") Optional runningSentinel, @JsonProperty("extraCmdLineArgs") List extraCmdLineArgs, @JsonProperty("loggingTag") Optional loggingTag, @JsonProperty("loggingExtraFields") Map loggingExtraFields, @JsonProperty("sigKillProcessesAfterMillis") Optional sigKillProcessesAfterMillis, - @JsonProperty("maxTaskThreads") Optional maxTaskThreads, @JsonProperty("preserveTaskSandboxAfterFinish") Optional preserveTaskSandboxAfterFinish, - @JsonProperty("loggingS3Bucket") Optional loggingS3Bucket, @JsonProperty("maxOpenFiles") Optional maxOpenFiles, + @JsonProperty("maxTaskThreads") Optional maxTaskThreads, @JsonProperty("preserveTaskSandboxAfterFinish") Optional preserveTaskSandboxAfterFinish, @JsonProperty("maxOpenFiles") Optional maxOpenFiles, @JsonProperty("skipLogrotateAndCompress") Optional skipLogrotateAndCompress, @JsonProperty("s3ArtifactSignatures") Optional> s3ArtifactSignatures, @JsonProperty("logrotateFrequency") Optional logrotateFrequency) { this.cmd = cmd; @@ -56,7 +54,6 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti this.sigKillProcessesAfterMillis = sigKillProcessesAfterMillis; this.maxTaskThreads = maxTaskThreads; this.preserveTaskSandboxAfterFinish = preserveTaskSandboxAfterFinish; - this.loggingS3Bucket = loggingS3Bucket; this.maxOpenFiles = maxOpenFiles; this.skipLogrotateAndCompress = skipLogrotateAndCompress; this.s3ArtifactSignatures = s3ArtifactSignatures; @@ -65,7 +62,7 @@ public ExecutorData(@JsonProperty("cmd") String cmd, @JsonProperty("embeddedArti public ExecutorDataBuilder toBuilder() { return new ExecutorDataBuilder(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, runningSentinel, user, extraCmdLineArgs, loggingTag, - loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, loggingS3Bucket, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency); + loggingExtraFields, sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency); } @ApiModelProperty(required=true, value="Command for the custom executor to run") @@ -133,11 +130,6 @@ public Optional getPreserveTaskSandboxAfterFinish() { return preserveTaskSandboxAfterFinish; } - @ApiModelProperty(required=false, value="Override the default bucket used by the S3Uploader to store log files") - public Optional getLoggingS3Bucket() { - return loggingS3Bucket; - } - @ApiModelProperty(required=false, value="Maximum number of open files the task process is allowed") public Optional getMaxOpenFiles() { return maxOpenFiles; @@ -174,7 +166,6 @@ public String toString() { .add("sigKillProcessesAfterMillis", sigKillProcessesAfterMillis) .add("maxTaskThreads", maxTaskThreads) .add("preserveTaskSandboxAfterFinish", preserveTaskSandboxAfterFinish) - .add("loggingS3Bucket", loggingS3Bucket) .add("maxOpenFiles", maxOpenFiles) .add("skipLogrotateAndCompress", skipLogrotateAndCompress) .add("s3ArtifactSignatures", s3ArtifactSignatures) diff --git a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java index cde6fecc23..262d2cdd61 100644 --- a/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/deploy/ExecutorDataBuilder.java @@ -21,7 +21,6 @@ public class ExecutorDataBuilder { private Optional sigKillProcessesAfterMillis; private Optional maxTaskThreads; private Optional preserveTaskSandboxAfterFinish; - private Optional loggingS3Bucket; private Optional maxOpenFiles; private Optional skipLogrotateAndCompress; private Optional> s3ArtifactSignatures; @@ -29,7 +28,7 @@ public class ExecutorDataBuilder { public ExecutorDataBuilder(String cmd, List embeddedArtifacts, List externalArtifacts, List s3Artifacts, List successfulExitCodes, Optional runningSentinel, Optional user, List extraCmdLineArgs, Optional loggingTag, Map loggingExtraFields, - Optional sigKillProcessesAfterMillis, Optional maxTaskThreads, Optional preserveTaskSandboxAfterFinish, Optional loggingS3Bucket, + Optional sigKillProcessesAfterMillis, Optional maxTaskThreads, Optional preserveTaskSandboxAfterFinish, Optional maxOpenFiles, Optional skipLogrotateAndCompress, Optional> s3ArtifactSignatures, Optional logrotateFrequency) { this.cmd = cmd; this.embeddedArtifacts = embeddedArtifacts; @@ -44,7 +43,6 @@ public ExecutorDataBuilder(String cmd, List embeddedArtifacts, this.sigKillProcessesAfterMillis = sigKillProcessesAfterMillis; this.maxTaskThreads = maxTaskThreads; this.preserveTaskSandboxAfterFinish = preserveTaskSandboxAfterFinish; - this.loggingS3Bucket = loggingS3Bucket; this.maxOpenFiles = maxOpenFiles; this.skipLogrotateAndCompress = skipLogrotateAndCompress; this.s3ArtifactSignatures = s3ArtifactSignatures; @@ -57,7 +55,7 @@ public ExecutorDataBuilder() { public ExecutorData build() { return new ExecutorData(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, user, runningSentinel, extraCmdLineArgs, loggingTag, loggingExtraFields, - sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, loggingS3Bucket, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency); + sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency); } public Optional getLoggingTag() { @@ -177,15 +175,6 @@ public ExecutorDataBuilder setPreserveTaskSandboxAfterFinish(Optional p return this; } - public Optional getLoggingS3Bucket() { - return loggingS3Bucket; - } - - public ExecutorDataBuilder setLoggingS3Bucket(Optional loggingS3Bucket) { - this.loggingS3Bucket = loggingS3Bucket; - return this; - } - public Optional getMaxOpenFiles() { return maxOpenFiles; } @@ -238,7 +227,6 @@ public String toString() { ", sigKillProcessesAfterMillis=" + sigKillProcessesAfterMillis + ", maxTaskThreads=" + maxTaskThreads + ", preserveTaskSandboxAfterFinish=" + preserveTaskSandboxAfterFinish + - ", loggingS3Bucket=" + loggingS3Bucket + ", maxOpenFiles=" + maxOpenFiles + ", skipLogrotateAndCompress=" + skipLogrotateAndCompress + ", s3ArtifactSignatures=" + s3ArtifactSignatures + diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java index e13c01471c..942323cc9d 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3FormatHelper.java @@ -13,19 +13,21 @@ import com.google.common.collect.Sets; public class SingularityS3FormatHelper { + public static final String DEFAULT_GROUP_NAME = "default"; private static final List DISALLOWED_FOR_TASK = ImmutableList.of("%index", "%s", "%filename", "%fileext"); private static final List DISALLOWED_FOR_DEPLOY = ImmutableList.copyOf(Iterables.concat(DISALLOWED_FOR_TASK, ImmutableList.of("%host"))); private static final List DISALLOWED_FOR_REQUEST = ImmutableList.copyOf(Iterables.concat(DISALLOWED_FOR_DEPLOY, ImmutableList.of("%tag", "%deployId"))); - public static String getS3KeyFormat(String s3KeyFormat, String requestId) { + public static String getS3KeyFormat(String s3KeyFormat, String requestId, String group) { s3KeyFormat = s3KeyFormat.replace("%requestId", requestId); + s3KeyFormat = s3KeyFormat.replace("%group", group); return s3KeyFormat; } - public static String getS3KeyFormat(String s3KeyFormat, String requestId, String deployId, Optional loggingTag) { - s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId); + public static String getS3KeyFormat(String s3KeyFormat, String requestId, String deployId, Optional loggingTag, String group) { + s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group); s3KeyFormat = s3KeyFormat.replace("%tag", loggingTag.or("")); s3KeyFormat = s3KeyFormat.replace("%deployId", deployId); @@ -33,8 +35,8 @@ public static String getS3KeyFormat(String s3KeyFormat, String requestId, String return s3KeyFormat; } - public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional loggingTag) { - s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag); + public static String getS3KeyFormat(String s3KeyFormat, SingularityTaskId taskId, Optional loggingTag, String group) { + s3KeyFormat = getS3KeyFormat(s3KeyFormat, taskId.getRequestId(), taskId.getDeployId(), loggingTag, group); s3KeyFormat = s3KeyFormat.replace("%host", taskId.getSanitizedHost()); s3KeyFormat = s3KeyFormat.replace("%taskId", taskId.toString()); @@ -118,8 +120,8 @@ private static String getDayOrMonth(int value) { return String.format("%02d", value); } - public static Collection getS3KeyPrefixes(String s3KeyFormat, String requestId, String deployId, Optional tag, long start, long end) { - String keyFormat = getS3KeyFormat(s3KeyFormat, requestId, deployId, tag); + public static Collection getS3KeyPrefixes(String s3KeyFormat, String requestId, String deployId, Optional tag, long start, long end, String group) { + String keyFormat = getS3KeyFormat(s3KeyFormat, requestId, deployId, tag, group); keyFormat = trimTaskId(keyFormat, requestId + "-" + deployId); @@ -136,8 +138,8 @@ private static String trimTaskId(String s3KeyFormat, String replaceWith) { return s3KeyFormat; } - public static Collection getS3KeyPrefixes(String s3KeyFormat, String requestId, long start, long end) { - s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId); + public static Collection getS3KeyPrefixes(String s3KeyFormat, String requestId, long start, long end, String group) { + s3KeyFormat = getS3KeyFormat(s3KeyFormat, requestId, group); s3KeyFormat = trimTaskId(s3KeyFormat, requestId); @@ -206,8 +208,8 @@ private static Collection getS3KeyPrefixes(String s3KeyFormat, List getS3KeyPrefixes(String s3KeyFormat, SingularityTaskId taskId, Optional tag, long start, long end) { - String keyFormat = getS3KeyFormat(s3KeyFormat, taskId, tag); + public static Collection getS3KeyPrefixes(String s3KeyFormat, SingularityTaskId taskId, Optional tag, long start, long end, String group) { + String keyFormat = getS3KeyFormat(s3KeyFormat, taskId, tag, group); return getS3KeyPrefixes(keyFormat, DISALLOWED_FOR_TASK, start, end); } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3Log.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3Log.java index 25af265e23..ba8a08d596 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3Log.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3Log.java @@ -7,28 +7,16 @@ import com.wordnik.swagger.annotations.ApiModelProperty; @ApiModel( description = "Represents a task sandbox file that was uploaded to S3" ) -public class SingularityS3Log { - public static final String LOG_START_S3_ATTR = "starttime"; - public static final String LOG_END_S3_ATTR = "endtime"; - +public class SingularityS3Log extends SingularityS3LogMetadata { private final String getUrl; - private final String key; - private final long lastModified; - private final long size; private final String downloadUrl; - private final Optional startTime; - private final Optional endTime; @JsonCreator public SingularityS3Log(@JsonProperty("getUrl") String getUrl, @JsonProperty("key") String key, @JsonProperty("lastModified") long lastModified, @JsonProperty("size") long size, @JsonProperty("downloadUrl") String downloadUrl, @JsonProperty("startTime") Optional startTime, @JsonProperty("endTime") Optional endTime) { + super(key, lastModified, size, startTime, endTime); this.getUrl = getUrl; - this.key = key; - this.lastModified = lastModified; - this.size = size; this.downloadUrl = downloadUrl; - this.startTime = startTime; - this.endTime = endTime; } @ApiModelProperty("URL to file in S3") @@ -36,46 +24,16 @@ public String getGetUrl() { return getUrl; } - @ApiModelProperty("S3 key") - public String getKey() { - return key; - } - - @ApiModelProperty("Last modified time") - public long getLastModified() { - return lastModified; - } - - @ApiModelProperty("File size (in bytes)") - public long getSize() { - return size; - } - @ApiModelProperty("URL to file in S3 containing headers that will force file to be downloaded instead of viewed") public String getDownloadUrl() { return downloadUrl; } - @ApiModelProperty("Time the log file started being written to") - public Optional getStartTime() { - return startTime; - } - - @ApiModelProperty("Time the log file was finished being written to") - public Optional getEndTime() { - return endTime; - } - @Override public String toString() { return "SingularityS3Log{" + "getUrl='" + getUrl + '\'' + - ", key='" + key + '\'' + - ", lastModified=" + lastModified + - ", size=" + size + ", downloadUrl='" + downloadUrl + '\'' + - ", startTime=" + startTime + - ", endTime=" + endTime + - '}'; + "} " + super.toString(); } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3LogMetadata.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3LogMetadata.java new file mode 100644 index 0000000000..8d85319624 --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3LogMetadata.java @@ -0,0 +1,63 @@ +package com.hubspot.singularity; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; +import com.wordnik.swagger.annotations.ApiModelProperty; + +public class SingularityS3LogMetadata { + public static final String LOG_START_S3_ATTR = "starttime"; + public static final String LOG_END_S3_ATTR = "endtime"; + + private final String key; + private final long lastModified; + private final long size; + private final Optional startTime; + private final Optional endTime; + + @JsonCreator + public SingularityS3LogMetadata(@JsonProperty("key") String key, @JsonProperty("lastModified") long lastModified, @JsonProperty("size") long size, + @JsonProperty("startTime") Optional startTime, @JsonProperty("endTime") Optional endTime) { + this.key = key; + this.lastModified = lastModified; + this.size = size; + this.startTime = startTime; + this.endTime = endTime; + } + + @ApiModelProperty("S3 key") + public String getKey() { + return key; + } + + @ApiModelProperty("Last modified time") + public long getLastModified() { + return lastModified; + } + + @ApiModelProperty("File size (in bytes)") + public long getSize() { + return size; + } + + @ApiModelProperty("Time the log file started being written to") + public Optional getStartTime() { + return startTime; + } + + @ApiModelProperty("Time the log file was finished being written to") + public Optional getEndTime() { + return endTime; + } + + @Override + public String toString() { + return "SingularityS3Log{" + + "key='" + key + '\'' + + ", lastModified=" + lastModified + + ", size=" + size + + ", startTime=" + startTime + + ", endTime=" + endTime + + '}'; + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3UploaderFile.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3UploaderFile.java new file mode 100644 index 0000000000..7ff506949b --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3UploaderFile.java @@ -0,0 +1,78 @@ +package com.hubspot.singularity; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; + +public class SingularityS3UploaderFile { + private final String filename; + private final Optional s3UploaderBucket; + private final Optional s3UploaderKeyPattern; + private final Optional s3UploaderFilenameHint; + private final Optional directory; + private final Optional s3StorageClass; + private final Optional applyS3StorageClassAfterBytes; + + @JsonCreator + public static SingularityS3UploaderFile fromString(String value) { + return new SingularityS3UploaderFile(value, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent()); + } + + @JsonCreator + public SingularityS3UploaderFile(@JsonProperty("filename") String filename, + @JsonProperty("s3UploaderBucket") Optional s3UploaderBucket, + @JsonProperty("s3UploaderKeyPattern") Optional s3UploaderKeyPattern, + @JsonProperty("s3UploaderFilenameHint") Optional s3UploaderFilenameHint, + @JsonProperty("directory") Optional directory, + @JsonProperty("s3StorageClass") Optional s3StorageClass, + @JsonProperty("applyS3StorageClassAfterBytes") Optional applyS3StorageClassAfterBytes) { + this.filename = filename; + this.s3UploaderBucket = s3UploaderBucket; + this.s3UploaderKeyPattern = s3UploaderKeyPattern; + this.s3UploaderFilenameHint = s3UploaderFilenameHint; + this.directory = directory; + this.s3StorageClass = s3StorageClass; + this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; + } + + public String getFilename() { + return filename; + } + + public Optional getS3UploaderBucket() { + return s3UploaderBucket; + } + + public Optional getS3UploaderKeyPattern() { + return s3UploaderKeyPattern; + } + + public Optional getS3UploaderFilenameHint() { + return s3UploaderFilenameHint; + } + + public Optional getDirectory() { + return directory; + } + + public Optional getS3StorageClass() { + return s3StorageClass; + } + + public Optional getApplyS3StorageClassAfterBytes() { + return applyS3StorageClassAfterBytes; + } + + @Override + public String toString() { + return "SingularityS3UploaderFile{" + + "filename='" + filename + '\'' + + ", s3UploaderBucket=" + s3UploaderBucket + + ", s3UploaderKeyPattern=" + s3UploaderKeyPattern + + ", s3UploaderFilenameHint=" + s3UploaderFilenameHint + + ", directory=" + directory + + ", s3StorageClass=" + s3StorageClass + + ", applyS3StorageClassAfterBytes=" + applyS3StorageClassAfterBytes + + '}'; + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskExecutorData.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskExecutorData.java new file mode 100644 index 0000000000..6a24568529 --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskExecutorData.java @@ -0,0 +1,138 @@ +package com.hubspot.singularity; + +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Optional; +import com.hubspot.deploy.EmbeddedArtifact; +import com.hubspot.deploy.ExecutorData; +import com.hubspot.deploy.ExternalArtifact; +import com.hubspot.deploy.S3Artifact; +import com.hubspot.deploy.S3ArtifactSignature; +import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; + +public class SingularityTaskExecutorData extends ExecutorData { + private final List s3UploaderAdditionalFiles; + private final String defaultS3Bucket; + private final String s3UploaderKeyPattern; + private final String serviceLog; + private final String serviceFinishedTailLog; + private final Optional requestGroup; + private final Optional s3StorageClass; + private final Optional applyS3StorageClassAfterBytes; + + public SingularityTaskExecutorData(ExecutorData executorData, List s3UploaderAdditionalFiles, String defaultS3Bucket, String s3UploaderKeyPattern, + String serviceLog, String serviceFinishedTailLog, Optional requestGroup, Optional s3StorageClass, Optional applyS3StorageClassAfterBytes) { + this(executorData.getCmd(), + executorData.getEmbeddedArtifacts(), + executorData.getExternalArtifacts(), + executorData.getS3Artifacts(), + executorData.getSuccessfulExitCodes(), + executorData.getUser(), + executorData.getRunningSentinel(), + executorData.getExtraCmdLineArgs(), + executorData.getLoggingTag(), + executorData.getLoggingExtraFields(), + executorData.getSigKillProcessesAfterMillis(), + executorData.getMaxTaskThreads(), + executorData.getPreserveTaskSandboxAfterFinish(), + executorData.getMaxOpenFiles(), + executorData.getSkipLogrotateAndCompress(), + executorData.getS3ArtifactSignatures(), + executorData.getLogrotateFrequency(), + s3UploaderAdditionalFiles, + defaultS3Bucket, + s3UploaderKeyPattern, + serviceLog, + serviceFinishedTailLog, + requestGroup, + s3StorageClass, + applyS3StorageClassAfterBytes); + } + + @JsonCreator + public SingularityTaskExecutorData(@JsonProperty("cmd") String cmd, + @JsonProperty("embeddedArtifacts") List embeddedArtifacts, + @JsonProperty("externalArtifacts") List externalArtifacts, + @JsonProperty("s3Artifacts") List s3Artifacts, + @JsonProperty("successfulExitCodes") List successfulExitCodes, + @JsonProperty("user") Optional user, + @JsonProperty("runningSentinel") Optional runningSentinel, + @JsonProperty("extraCmdLineArgs") List extraCmdLineArgs, + @JsonProperty("loggingTag") Optional loggingTag, + @JsonProperty("loggingExtraFields") Map loggingExtraFields, + @JsonProperty("sigKillProcessesAfterMillis") Optional sigKillProcessesAfterMillis, + @JsonProperty("maxTaskThreads") Optional maxTaskThreads, + @JsonProperty("preserveTaskSandboxAfterFinish") Optional preserveTaskSandboxAfterFinish, + @JsonProperty("maxOpenFiles") Optional maxOpenFiles, + @JsonProperty("skipLogrotateAndCompress") Optional skipLogrotateAndCompress, + @JsonProperty("s3ArtifactSignatures") Optional> s3ArtifactSignatures, + @JsonProperty("logrotateFrequency") Optional logrotateFrequency, + @JsonProperty("s3UploaderAdditionalFiles") List s3UploaderAdditionalFiles, + @JsonProperty("defaultS3Bucket") String defaultS3Bucket, + @JsonProperty("s3UploaderKeyPattern") String s3UploaderKeyPattern, + @JsonProperty("serviceLog") String serviceLog, + @JsonProperty("serviceFinishedTailLog") String serviceFinishedTailLog, + @JsonProperty("requestGroup") Optional requestGroup, + @JsonProperty("s3StorageClass") Optional s3StorageClass, + @JsonProperty("applyS3StorageClassAfterBytes") Optional applyS3StorageClassAfterBytes) { + super(cmd, embeddedArtifacts, externalArtifacts, s3Artifacts, successfulExitCodes, user, runningSentinel, extraCmdLineArgs, loggingTag, loggingExtraFields, + sigKillProcessesAfterMillis, maxTaskThreads, preserveTaskSandboxAfterFinish, maxOpenFiles, skipLogrotateAndCompress, s3ArtifactSignatures, logrotateFrequency); + this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles; + this.defaultS3Bucket = defaultS3Bucket; + this.s3UploaderKeyPattern = s3UploaderKeyPattern; + this.serviceLog = serviceLog; + this.serviceFinishedTailLog = serviceFinishedTailLog; + this.requestGroup = requestGroup; + this.s3StorageClass = s3StorageClass; + this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; + } + + public List getS3UploaderAdditionalFiles() { + return s3UploaderAdditionalFiles; + } + + public String getDefaultS3Bucket() { + return defaultS3Bucket; + } + + public String getS3UploaderKeyPattern() { + return s3UploaderKeyPattern; + } + + public String getServiceLog() { + return serviceLog; + } + + public String getServiceFinishedTailLog() { + return serviceFinishedTailLog; + } + + public Optional getRequestGroup() { + return requestGroup; + } + + public Optional getS3StorageClass() { + return s3StorageClass; + } + + public Optional getApplyS3StorageClassAfterBytes() { + return applyS3StorageClassAfterBytes; + } + + @Override + public String toString() { + return "SingularityTaskExecutorData{" + + "s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles + + ", defaultS3Bucket='" + defaultS3Bucket + '\'' + + ", s3UploaderKeyPattern='" + s3UploaderKeyPattern + '\'' + + ", serviceLog='" + serviceLog + '\'' + + ", serviceFinishedTailLog='" + serviceFinishedTailLog + '\'' + + ", requestGroup=" + requestGroup + + ", s3StorageClass=" + s3StorageClass + + ", applyS3StorageClassAfterBytes=" + applyS3StorageClassAfterBytes + + "} " + super.toString(); + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/ContinuationToken.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/ContinuationToken.java new file mode 100644 index 0000000000..cae18b47de --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/ContinuationToken.java @@ -0,0 +1,58 @@ +package com.hubspot.singularity.api; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.wordnik.swagger.annotations.ApiModelProperty; + +public class ContinuationToken { + private final String value; + private final boolean lastPage; + + @JsonCreator + public ContinuationToken(@JsonProperty("value") String value, @JsonProperty("lastPage") boolean lastPage) { + this.value = value; + this.lastPage = lastPage; + } + + @ApiModelProperty(required=true, value="S3 continuation token specific to a bucket + prefix being searched") + public String getValue() { + return value; + } + + @ApiModelProperty(required=true, value="If true, there are no further results for this bucket + prefix") + public boolean isLastPage() { + return lastPage; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + ContinuationToken token = (ContinuationToken) o; + + if (lastPage != token.lastPage) { + return false; + } + return value != null ? value.equals(token.value) : token.value == null; + } + + @Override + public int hashCode() { + int result = value != null ? value.hashCode() : 0; + result = 31 * result + (lastPage ? 1 : 0); + return result; + } + + @Override + public String toString() { + return "ContinuationToken{" + + "value='" + value + '\'' + + ", lastPage=" + lastPage + + '}'; + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchRequest.java new file mode 100644 index 0000000000..978c1606e3 --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchRequest.java @@ -0,0 +1,100 @@ +package com.hubspot.singularity.api; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Objects; +import com.google.common.base.Optional; +import com.wordnik.swagger.annotations.ApiModelProperty; + +public class SingularityS3SearchRequest { + private final Map> requestsAndDeploys; + private final List taskIds; + private final Optional start; + private final Optional end; + private final boolean excludeMetadata; + private final boolean listOnly; + private final Optional maxPerPage; + private final Map continuationTokens; + + @JsonCreator + + public SingularityS3SearchRequest(@JsonProperty("requestsAndDeploys") Map> requestsAndDeploys, + @JsonProperty("taskIds") List taskIds, + @JsonProperty("start") Optional start, + @JsonProperty("end") Optional end, + @JsonProperty("excludeMetadata") boolean excludeMetadata, + @JsonProperty("listOnly") boolean listOnly, + @JsonProperty("maxPerPage") Optional maxPerPage, + @JsonProperty("continuationTokens") Map continuationTokens) { + this.requestsAndDeploys = Objects.firstNonNull(requestsAndDeploys, Collections.>emptyMap()); + this.taskIds = Objects.firstNonNull(taskIds, Collections.emptyList()); + this.start = start; + this.end = end; + this.excludeMetadata = excludeMetadata; + this.listOnly = listOnly; + this.maxPerPage = maxPerPage; + this.continuationTokens = Objects.firstNonNull(continuationTokens, Collections.emptyMap()); + } + + @ApiModelProperty(required=false, value="A map of request IDs to a list of deploy ids to search") + public Map> getRequestsAndDeploys() { + return requestsAndDeploys; + } + + @ApiModelProperty(required=false, value="A list of task IDs to search for") + public List getTaskIds() { + return taskIds; + } + + @ApiModelProperty(required=false, value="Start timestamp (millis, 13 digit)") + public Optional getStart() { + return start; + } + + @ApiModelProperty(required=false, value="End timestamp (millis, 13 digit)") + public Optional getEnd() { + return end; + } + + @ApiModelProperty(required=false, value="if true, do not query for custom start/end time metadata") + public boolean isExcludeMetadata() { + return excludeMetadata; + } + + @ApiModelProperty(required=false, value="If true, do not generate download/get urls, only list objects") + public boolean isListOnly() { + return listOnly; + } + + /** + * NOTE: maxPerPage is not a guaranteed value. It is possible to get as many as (maxPerPage * 2 - 1) results + * when using the paginated search endpoint + */ + @ApiModelProperty(required=false, value="Target number of results to return") + public Optional getMaxPerPage() { + return maxPerPage; + } + + @ApiModelProperty(required=false, value="S3 continuation tokens, return these to Singularity to continue searching subsequent pages of results") + public Map getContinuationTokens() { + return continuationTokens; + } + + @Override + public String toString() { + return "SingularityS3SearchRequest{" + + "requestsAndDeploys=" + requestsAndDeploys + + ", taskIds=" + taskIds + + ", start=" + start + + ", end=" + end + + ", excludeMetadata=" + excludeMetadata + + ", listOnly=" + listOnly + + ", maxPerPage=" + maxPerPage + + ", continuationTokens=" + continuationTokens + + '}'; + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchResult.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchResult.java new file mode 100644 index 0000000000..2e7da3d7ec --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityS3SearchResult.java @@ -0,0 +1,53 @@ +package com.hubspot.singularity.api; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.hubspot.singularity.SingularityS3LogMetadata; +import com.wordnik.swagger.annotations.ApiModelProperty; + +public class SingularityS3SearchResult { + private final Map continuationTokens; + private final boolean lastPage; + private final List results; + + public static SingularityS3SearchResult empty() { + return new SingularityS3SearchResult(Collections.emptyMap(), false, Collections.emptyList()); + } + + @JsonCreator + public SingularityS3SearchResult(@JsonProperty("continuationTokens") Map continuationTokens, + @JsonProperty("lastPage") boolean lastPage, + @JsonProperty("results") List results) { + this.continuationTokens = continuationTokens; + this.lastPage = lastPage; + this.results = results; + } + + @ApiModelProperty(required=false, value="S3 continuation tokens, return these to Singularity to continue searching subsequent pages of results") + public Map getContinuationTokens() { + return continuationTokens; + } + + @ApiModelProperty(required=true, value="If true, there are no further results for any bucket + prefix being searched") + public boolean isLastPage() { + return lastPage; + } + + @ApiModelProperty("List of S3 log metadata") + public List getResults() { + return results; + } + + @Override + public String toString() { + return "SingularityS3SearchResult{" + + "continuationTokens=" + continuationTokens + + ", lastPage=" + lastPage + + ", results=" + results + + '}'; + } +} diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index ccf33feb3c..c11c9f08d3 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -33,10 +33,6 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration { @JsonProperty private String executorBashLog = "executor.bash.log"; - @NotEmpty - @JsonProperty - private String serviceLog = "service.log"; - @NotEmpty @JsonProperty private String defaultRunAsUser; @@ -127,28 +123,10 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration { @JsonProperty private List logrotateAdditionalFiles = Collections.emptyList(); - /** - * Extra files to backup to S3 besides the service log. - */ - @NotNull - @JsonProperty - private List s3UploaderAdditionalFiles = Collections.emptyList(); - @Min(1) @JsonProperty private int tailLogLinesToSave = 2500; - @NotEmpty - @JsonProperty - private String serviceFinishedTailLog = "tail_of_finished_service.log"; - - @NotEmpty - @JsonProperty - private String s3UploaderKeyPattern = "%requestId/%Y/%m/%taskId_%index-%s-%filename"; - - @JsonProperty - private String s3UploaderBucket; - @JsonProperty private boolean useLocalDownloadService = false; @@ -234,14 +212,6 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration { @JsonProperty private SingularityExecutorLogrotateFrequency logrotateFrequency = SingularityExecutorLogrotateFrequency.DAILY; - @NotNull - @JsonProperty - private Optional s3StorageClass = Optional.absent(); - - @NotNull - @JsonProperty - private Optional applyS3StorageClassAfterBytes = Optional.absent(); - @NotEmpty @JsonProperty private String cronDirectory = "/etc/cron.d"; @@ -277,14 +247,6 @@ public void setLogrotateAdditionalFiles(List getS3UploaderAdditionalFiles() { - return s3UploaderAdditionalFiles; - } - - public void setS3UploaderAdditionalFiles(List s3UploaderAdditionalFiles) { - this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles; - } - public String getExecutorJavaLog() { return executorJavaLog; } @@ -293,10 +255,6 @@ public String getExecutorBashLog() { return executorBashLog; } - public String getServiceLog() { - return serviceLog; - } - public String getDefaultRunAsUser() { return defaultRunAsUser; } @@ -389,10 +347,6 @@ public int getTailLogLinesToSave() { return tailLogLinesToSave; } - public String getServiceFinishedTailLog() { - return serviceFinishedTailLog; - } - public boolean isUseLocalDownloadService() { return useLocalDownloadService; } @@ -426,10 +380,6 @@ public void setExecutorBashLog(String executorBashLog) { this.executorBashLog = executorBashLog; } - public void setServiceLog(String serviceLog) { - this.serviceLog = serviceLog; - } - public void setDefaultRunAsUser(String defaultRunAsUser) { this.defaultRunAsUser = defaultRunAsUser; } @@ -514,26 +464,6 @@ public void setTailLogLinesToSave(int tailLogLinesToSave) { this.tailLogLinesToSave = tailLogLinesToSave; } - public void setServiceFinishedTailLog(String serviceFinishedTailLog) { - this.serviceFinishedTailLog = serviceFinishedTailLog; - } - - public String getS3UploaderKeyPattern() { - return s3UploaderKeyPattern; - } - - public void setS3UploaderKeyPattern(String s3UploaderKeyPattern) { - this.s3UploaderKeyPattern = s3UploaderKeyPattern; - } - - public String getS3UploaderBucket() { - return s3UploaderBucket; - } - - public void setS3UploaderBucket(String s3UploaderBucket) { - this.s3UploaderBucket = s3UploaderBucket; - } - public void setUseLocalDownloadService(boolean useLocalDownloadService) { this.useLocalDownloadService = useLocalDownloadService; } @@ -706,83 +636,60 @@ public void setLogrotateCompressionSettings(LogrotateCompressionSettings logrota this.logrotateCompressionSettings = logrotateCompressionSettings; } - public Optional getS3StorageClass() { - return s3StorageClass; - } - - public void setS3StorageClass(Optional s3StorageClass) { - this.s3StorageClass = s3StorageClass; - } - - public Optional getApplyS3StorageClassAfterBytes() { - return applyS3StorageClassAfterBytes; - } - - public void setApplyS3StorageClassAfterBytes(Optional applyS3StorageClassAfterBytes) { - this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; - } - @Override public String toString() { return "SingularityExecutorConfiguration{" + - "executorJavaLog='" + executorJavaLog + '\'' + - ", executorBashLog='" + executorBashLog + '\'' + - ", serviceLog='" + serviceLog + '\'' + - ", defaultRunAsUser='" + defaultRunAsUser + '\'' + - ", taskAppDirectory='" + taskAppDirectory + '\'' + - ", shutdownTimeoutWaitMillis=" + shutdownTimeoutWaitMillis + - ", idleExecutorShutdownWaitMillis=" + idleExecutorShutdownWaitMillis + - ", stopDriverAfterMillis=" + stopDriverAfterMillis + - ", globalTaskDefinitionDirectory='" + globalTaskDefinitionDirectory + '\'' + - ", globalTaskDefinitionSuffix='" + globalTaskDefinitionSuffix + '\'' + - ", hardKillAfterMillis=" + hardKillAfterMillis + - ", killThreads=" + killThreads + - ", threadCheckThreads=" + threadCheckThreads + - ", checkThreadsEveryMillis=" + checkThreadsEveryMillis + - ", maxTaskMessageLength=" + maxTaskMessageLength + - ", logrotateCommand='" + logrotateCommand + '\'' + - ", logrotateStateFile='" + logrotateStateFile + '\'' + - ", logrotateConfDirectory='" + logrotateConfDirectory + '\'' + - ", logrotateToDirectory='" + logrotateToDirectory + '\'' + - ", logrotateMaxageDays=" + logrotateMaxageDays + - ", logrotateCount=" + logrotateCount + - ", logrotateDateformat='" + logrotateDateformat + '\'' + - ", logrotateExtrasDateformat='" + logrotateExtrasDateformat + '\'' + - ", logrotateCompressionSettings=" + logrotateCompressionSettings + - ", logrotateAdditionalFiles=" + logrotateAdditionalFiles + - ", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles + - ", tailLogLinesToSave=" + tailLogLinesToSave + - ", serviceFinishedTailLog='" + serviceFinishedTailLog + '\'' + - ", s3UploaderKeyPattern='" + s3UploaderKeyPattern + '\'' + - ", s3UploaderBucket='" + s3UploaderBucket + '\'' + - ", useLocalDownloadService=" + useLocalDownloadService + - ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + - ", localDownloadServiceMaxConnections=" + localDownloadServiceMaxConnections + - ", maxTaskThreads=" + maxTaskThreads + - ", dockerPrefix='" + dockerPrefix + '\'' + - ", dockerStopTimeout=" + dockerStopTimeout + - ", cgroupsMesosCpuTasksFormat='" + cgroupsMesosCpuTasksFormat + '\'' + - ", procCgroupFormat='" + procCgroupFormat + '\'' + - ", switchUserCommandFormat='" + switchUserCommandFormat + '\'' + - ", artifactSignatureVerificationCommand=" + artifactSignatureVerificationCommand + - ", failTaskOnInvalidArtifactSignature=" + failTaskOnInvalidArtifactSignature + - ", signatureVerifyOut='" + signatureVerifyOut + '\'' + - ", shellCommands=" + shellCommands + - ", shellCommandOutFile='" + shellCommandOutFile + '\'' + - ", shellCommandPidPlaceholder='" + shellCommandPidPlaceholder + '\'' + - ", shellCommandUserPlaceholder='" + shellCommandUserPlaceholder + '\'' + - ", shellCommandPidFile='" + shellCommandPidFile + '\'' + - ", shellCommandPrefix=" + shellCommandPrefix + - ", dockerClientTimeLimitSeconds=" + dockerClientTimeLimitSeconds + - ", dockerClientConnectionPoolSize=" + dockerClientConnectionPoolSize + - ", maxDockerPullAttempts=" + maxDockerPullAttempts + - ", dockerAuthConfig=" + dockerAuthConfig + - ", threadCheckerType=" + threadCheckerType + - ", logrotateFrequency=" + logrotateFrequency + - ", cronDirectory='" + cronDirectory + '\'' + - ", applyS3StorageClassAfterBytes='" + applyS3StorageClassAfterBytes + '\'' + - ", s3StorageClass='" + s3StorageClass + '\'' + - ", useFileAttributes=" + useFileAttributes + - '}'; + "executorJavaLog='" + executorJavaLog + '\'' + + ", executorBashLog='" + executorBashLog + '\'' + + ", defaultRunAsUser='" + defaultRunAsUser + '\'' + + ", taskAppDirectory='" + taskAppDirectory + '\'' + + ", shutdownTimeoutWaitMillis=" + shutdownTimeoutWaitMillis + + ", idleExecutorShutdownWaitMillis=" + idleExecutorShutdownWaitMillis + + ", stopDriverAfterMillis=" + stopDriverAfterMillis + + ", globalTaskDefinitionDirectory='" + globalTaskDefinitionDirectory + '\'' + + ", globalTaskDefinitionSuffix='" + globalTaskDefinitionSuffix + '\'' + + ", hardKillAfterMillis=" + hardKillAfterMillis + + ", killThreads=" + killThreads + + ", threadCheckThreads=" + threadCheckThreads + + ", checkThreadsEveryMillis=" + checkThreadsEveryMillis + + ", maxTaskMessageLength=" + maxTaskMessageLength + + ", logrotateCommand='" + logrotateCommand + '\'' + + ", logrotateStateFile='" + logrotateStateFile + '\'' + + ", logrotateConfDirectory='" + logrotateConfDirectory + '\'' + + ", logrotateToDirectory='" + logrotateToDirectory + '\'' + + ", logrotateMaxageDays=" + logrotateMaxageDays + + ", logrotateCount=" + logrotateCount + + ", logrotateDateformat='" + logrotateDateformat + '\'' + + ", logrotateExtrasDateformat='" + logrotateExtrasDateformat + '\'' + + ", logrotateCompressionSettings=" + logrotateCompressionSettings + + ", logrotateAdditionalFiles=" + logrotateAdditionalFiles + + ", tailLogLinesToSave=" + tailLogLinesToSave + + ", useLocalDownloadService=" + useLocalDownloadService + + ", localDownloadServiceTimeoutMillis=" + localDownloadServiceTimeoutMillis + + ", localDownloadServiceMaxConnections=" + localDownloadServiceMaxConnections + + ", maxTaskThreads=" + maxTaskThreads + + ", dockerPrefix='" + dockerPrefix + '\'' + + ", dockerStopTimeout=" + dockerStopTimeout + + ", cgroupsMesosCpuTasksFormat='" + cgroupsMesosCpuTasksFormat + '\'' + + ", procCgroupFormat='" + procCgroupFormat + '\'' + + ", switchUserCommandFormat='" + switchUserCommandFormat + '\'' + + ", artifactSignatureVerificationCommand=" + artifactSignatureVerificationCommand + + ", failTaskOnInvalidArtifactSignature=" + failTaskOnInvalidArtifactSignature + + ", signatureVerifyOut='" + signatureVerifyOut + '\'' + + ", shellCommands=" + shellCommands + + ", shellCommandOutFile='" + shellCommandOutFile + '\'' + + ", shellCommandPidPlaceholder='" + shellCommandPidPlaceholder + '\'' + + ", shellCommandUserPlaceholder='" + shellCommandUserPlaceholder + '\'' + + ", shellCommandPidFile='" + shellCommandPidFile + '\'' + + ", shellCommandPrefix=" + shellCommandPrefix + + ", dockerClientTimeLimitSeconds=" + dockerClientTimeLimitSeconds + + ", dockerClientConnectionPoolSize=" + dockerClientConnectionPoolSize + + ", maxDockerPullAttempts=" + maxDockerPullAttempts + + ", dockerAuthConfig=" + dockerAuthConfig + + ", threadCheckerType=" + threadCheckerType + + ", logrotateFrequency=" + logrotateFrequency + + ", cronDirectory='" + cronDirectory + '\'' + + ", useFileAttributes=" + useFileAttributes + + "} " + super.toString(); } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java deleted file mode 100644 index 4fce096b0b..0000000000 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorS3UploaderAdditionalFile.java +++ /dev/null @@ -1,76 +0,0 @@ -package com.hubspot.singularity.executor.config; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.Optional; - -public class SingularityExecutorS3UploaderAdditionalFile { - - private final String filename; - private final Optional s3UploaderBucket; - private final Optional s3UploaderKeyPattern; - private final Optional s3UploaderFilenameHint; - private final Optional directory; - private final Optional s3StorageClass; - private final Optional applyS3StorageClassAfterBytes; - - @JsonCreator - public static SingularityExecutorS3UploaderAdditionalFile fromString(String value) { - return new SingularityExecutorS3UploaderAdditionalFile(value, Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), Optional.absent(), - Optional. absent()); - } - - @JsonCreator - public SingularityExecutorS3UploaderAdditionalFile( - @JsonProperty("filename") String filename, - @JsonProperty("s3UploaderBucket") Optional s3UploaderBucket, - @JsonProperty("s3UploaderKeyPattern") Optional s3UploaderKeyPattern, - @JsonProperty("s3UploaderFilenameHint") Optional s3UploaderFilenameHint, - @JsonProperty("directory") Optional directory, - @JsonProperty("s3StorageClass") Optional s3StorageClass, - @JsonProperty("applyS3StorageClassAfterBytes") Optional applyS3StorageClassAfterBytes) { - this.filename = filename; - this.s3UploaderBucket = s3UploaderBucket; - this.s3UploaderKeyPattern = s3UploaderKeyPattern; - this.s3UploaderFilenameHint = s3UploaderFilenameHint; - this.directory = directory; - this.s3StorageClass = s3StorageClass; - this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; - } - - public String getFilename() { - return filename; - } - - public Optional getS3UploaderBucket() { - return s3UploaderBucket; - } - - public Optional getS3UploaderKeyPattern() { - return s3UploaderKeyPattern; - } - - public Optional getS3UploaderFilenameHint() { - return s3UploaderFilenameHint; - } - - public Optional getDirectory() { - return directory; - } - - public Optional getS3StorageClass() { - return s3StorageClass; - } - - public Optional getApplyS3StorageClassAfterBytes() { - return applyS3StorageClassAfterBytes; - } - - @Override - public String toString() { - return "SingularityExecutorS3UploaderAdditionalFile [filename=" + filename + ", s3UploaderBucket=" + s3UploaderBucket + ", s3UploaderKeyPattern=" + s3UploaderKeyPattern - + ", s3UploaderFilenameHint=" + s3UploaderFilenameHint + ", directory=" + directory + ", s3StorageClass=" + s3StorageClass + ", applyS3StorageClassAfterBytes=" + applyS3StorageClassAfterBytes - + "]"; - } - -} diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java index 4826a7eb0c..9ae65ddd93 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorTaskBuilder.java @@ -13,8 +13,8 @@ import com.google.inject.Inject; import com.google.inject.Singleton; import com.google.inject.name.Named; -import com.hubspot.deploy.ExecutorData; import com.hubspot.mesos.MesosUtils; +import com.hubspot.singularity.SingularityTaskExecutorData; import com.hubspot.singularity.executor.TemplateManager; import com.hubspot.singularity.executor.task.SingularityExecutorArtifactFetcher; import com.hubspot.singularity.executor.task.SingularityExecutorTask; @@ -72,21 +72,22 @@ public Logger buildTaskLogger(String taskId, String executorId) { } public SingularityExecutorTask buildTask(String taskId, ExecutorDriver driver, TaskInfo taskInfo, Logger log) { - ExecutorData executorData = readExecutorData(jsonObjectMapper, taskInfo); + SingularityTaskExecutorData taskExecutorData = readExecutorData(jsonObjectMapper, taskInfo); - SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, executorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), executorPid, - executorConfiguration.getServiceLog(), Files.getFileExtension(executorConfiguration.getServiceLog()), executorConfiguration.getTaskAppDirectory(), executorConfiguration.getExecutorBashLog(), executorConfiguration.getLogrotateStateFile(), executorConfiguration.getSignatureVerifyOut()); + SingularityExecutorTaskDefinition taskDefinition = new SingularityExecutorTaskDefinition(taskId, taskExecutorData, MesosUtils.getTaskDirectoryPath(taskId).toString(), executorPid, + taskExecutorData.getServiceLog(), Files.getFileExtension(taskExecutorData.getServiceLog()), taskExecutorData.getServiceFinishedTailLog(), executorConfiguration.getTaskAppDirectory(), + executorConfiguration.getExecutorBashLog(), executorConfiguration.getLogrotateStateFile(), executorConfiguration.getSignatureVerifyOut()); jsonObjectFileHelper.writeObject(taskDefinition, executorConfiguration.getTaskDefinitionPath(taskId), log); return new SingularityExecutorTask(driver, executorUtils, baseConfiguration, executorConfiguration, taskDefinition, executorPid, artifactFetcher, taskInfo, templateManager, log, jsonObjectFileHelper, dockerUtils, s3Configuration); } - private ExecutorData readExecutorData(ObjectMapper objectMapper, Protos.TaskInfo taskInfo) { + private SingularityTaskExecutorData readExecutorData(ObjectMapper objectMapper, Protos.TaskInfo taskInfo) { try { Preconditions.checkState(taskInfo.hasData(), "TaskInfo was missing executor data"); - return objectMapper.readValue(taskInfo.getData().toByteArray(), ExecutorData.class); + return objectMapper.readValue(taskInfo.getData().toByteArray(), SingularityTaskExecutorData.class); } catch (Exception e) { throw Throwables.propagate(e); } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index 217a2b7563..a0e3fabd88 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -122,7 +122,7 @@ public String getLogfileExtension() { } public String getLogfileName() { - return configuration.getServiceLog(); + return taskDefinition.getServiceLogFileName(); } public boolean isUseFileAttributes() { diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java index e48c48fa92..3c0f8ba523 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskDefinition.java @@ -7,32 +7,33 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; -import com.hubspot.deploy.ExecutorData; +import com.hubspot.singularity.SingularityTaskExecutorData; public class SingularityExecutorTaskDefinition { - private final ExecutorData executorData; + private final SingularityTaskExecutorData executorData; private final String taskId; private final Path taskDirectoryPath; private final String executorBashOut; private final String serviceLogOut; private final String serviceLogOutExtension; + private final String serviceFinishedTailLog; private final String taskAppDirectory; private final String logrotateStateFile; private final String executorPid; private final String signatureVerifyOut; @JsonCreator - public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") ExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, @JsonProperty("executorPid") String executorPid, - @JsonProperty("serviceLogOut") String serviceLogOut, @JsonProperty("serviceLogOutExtension") String serviceLogOutExtension, @JsonProperty("taskAppDirectory") String taskAppDirectory, @JsonProperty("executorBashOut") String executorBashOut, @JsonProperty("logrotateStateFilePath") String logrotateStateFile, @JsonProperty("signatureVerifyOut") String signatureVerifyOut) { + public SingularityExecutorTaskDefinition(@JsonProperty("taskId") String taskId, @JsonProperty("executorData") SingularityTaskExecutorData executorData, @JsonProperty("taskDirectory") String taskDirectory, @JsonProperty("executorPid") String executorPid, + @JsonProperty("serviceLogOut") String serviceLogOut, @JsonProperty("serviceLogOutExtension") String serviceLogOutExtension, @JsonProperty("serviceFinishedTailLog") String serviceFinishedTailLog, @JsonProperty("taskAppDirectory") String taskAppDirectory, @JsonProperty("executorBashOut") String executorBashOut, @JsonProperty("logrotateStateFilePath") String logrotateStateFile, @JsonProperty("signatureVerifyOut") String signatureVerifyOut) { this.executorData = executorData; this.taskId = taskId; this.taskDirectoryPath = Paths.get(taskDirectory); - this.executorPid = executorPid; this.executorBashOut = executorBashOut; this.serviceLogOut = serviceLogOut; this.serviceLogOutExtension = serviceLogOutExtension; + this.serviceFinishedTailLog = serviceFinishedTailLog; this.taskAppDirectory = taskAppDirectory; this.logrotateStateFile = logrotateStateFile; this.signatureVerifyOut = signatureVerifyOut; @@ -86,10 +87,29 @@ public String getServiceLogOut() { return getServiceLogOutPath().toString(); } + @JsonIgnore + public String getServiceLogFileName() { + return serviceLogOut; + } + + @JsonIgnore + public String getServiceFinishedTailLogFileName() { + return serviceFinishedTailLog; + } + public String getServiceLogOutExtension() { return serviceLogOutExtension; } + public String getServiceFinishedTailLog() { + return getServiceFinishedTailLogPath().toString(); + } + + @JsonIgnore + public Path getServiceFinishedTailLogPath() { + return taskDirectoryPath.resolve(serviceFinishedTailLog); + } + public String getTaskAppDirectory() { return getTaskAppDirectoryPath().toString(); } @@ -98,7 +118,7 @@ public String getLogrotateStateFile() { return getLogrotateStateFilePath().toString(); } - public ExecutorData getExecutorData() { + public SingularityTaskExecutorData getExecutorData() { return executorData; } @@ -125,8 +145,18 @@ public Optional getExecutorPidSafe() { @Override public String toString() { - return "SingularityExecutorTaskDefinition [executorData=" + executorData + ", taskId=" + taskId + ", taskDirectoryPath=" + taskDirectoryPath + ", executorBashOut=" + executorBashOut - + ", serviceLogOut=" + serviceLogOut + ", taskAppDirectory=" + taskAppDirectory + ", logrotateStateFile=" + logrotateStateFile + ", executorPid=" + executorPid + "]"; + return "SingularityExecutorTaskDefinition{" + + "executorData=" + executorData + + ", taskId='" + taskId + '\'' + + ", taskDirectoryPath=" + taskDirectoryPath + + ", executorBashOut='" + executorBashOut + '\'' + + ", serviceLogOut='" + serviceLogOut + '\'' + + ", serviceLogOutExtension='" + serviceLogOutExtension + '\'' + + ", serviceFinishedTailLog='" + serviceFinishedTailLog + '\'' + + ", taskAppDirectory='" + taskAppDirectory + '\'' + + ", logrotateStateFile='" + logrotateStateFile + '\'' + + ", executorPid='" + executorPid + '\'' + + ", signatureVerifyOut='" + signatureVerifyOut + '\'' + + '}'; } - } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index e39b9a8b28..de64f741a3 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -5,18 +5,20 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; import com.hubspot.singularity.executor.TemplateManager; import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; -import com.hubspot.singularity.executor.config.SingularityExecutorS3UploaderAdditionalFile; +import com.hubspot.singularity.SingularityS3UploaderFile; import com.hubspot.singularity.executor.models.LogrotateCronTemplateContext; import com.hubspot.singularity.executor.models.LogrotateTemplateContext; import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration; @@ -60,16 +62,23 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) { final Path serviceLogParent = serviceLogOutPath.getParent(); final Path logrotateDirectory = serviceLogParent.resolve(configuration.getLogrotateToDirectory()); - boolean result = writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.absent(), - Optional.absent(), finished, configuration.getS3StorageClass(), configuration.getApplyS3StorageClassAfterBytes()); - + List handledLogs = new ArrayList<>(); int index = 1; - for (SingularityExecutorS3UploaderAdditionalFile additionalFile : configuration.getS3UploaderAdditionalFiles()) { + boolean result = true; + + for (SingularityS3UploaderFile additionalFile : taskDefinition.getExecutorData().getS3UploaderAdditionalFiles()) { Path directory = additionalFile.getDirectory().isPresent() ? taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getDirectory().get()) : taskDefinition.getTaskDirectoryPath(); String fileGlob = additionalFile.getFilename() != null && additionalFile.getFilename().contains("*") ? additionalFile.getFilename() : String.format("%s*.[gb]z*", additionalFile.getFilename()); - result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("extra%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), - additionalFile.getS3UploaderKeyPattern(), finished, additionalFile.getS3StorageClass(), additionalFile.getApplyS3StorageClassAfterBytes()); + result = result && writeS3MetadataFile(additionalFile.getS3UploaderFilenameHint().or(String.format("file%d", index)), directory, fileGlob, additionalFile.getS3UploaderBucket(), additionalFile.getS3UploaderKeyPattern(), finished, + additionalFile.getS3StorageClass().or(taskDefinition.getExecutorData().getS3StorageClass()), additionalFile.getApplyS3StorageClassAfterBytes().or(taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes())); index++; + handledLogs.add(additionalFile.getFilename()); + } + + // Allow an additional file to override the upload settings for service.log + if (!handledLogs.contains(taskDefinition.getServiceLogFileName())) { + result = result && writeS3MetadataFile("default", logrotateDirectory, String.format("%s*.[gb]z*", taskDefinition.getServiceLogOutPath().getFileName()), Optional.absent(), Optional.absent(), finished, + taskDefinition.getExecutorData().getS3StorageClass(), taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes()); } return result; @@ -99,8 +108,8 @@ public boolean teardown() { if (!taskDefinition.shouldLogrotateLogFile()) { writeS3MetadataForNonLogRotatedFileSuccess = writeS3MetadataFile("unrotated", taskDefinition.getServiceLogOutPath().getParent(), - taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.absent(), Optional.absent(), true, configuration.getS3StorageClass(), - configuration.getApplyS3StorageClassAfterBytes()); + taskDefinition.getServiceLogOutPath().getFileName().toString(), Optional.absent(), Optional.absent(), true, + taskDefinition.getExecutorData().getS3StorageClass(), taskDefinition.getExecutorData().getApplyS3StorageClassAfterBytes()); } if (manualLogrotate()) { @@ -121,7 +130,7 @@ private void copyLogTail() { return; } - final Path tailOfLogPath = taskDefinition.getTaskDirectoryPath().resolve(configuration.getServiceFinishedTailLog()); + final Path tailOfLogPath = taskDefinition.getServiceFinishedTailLogPath(); if (Files.exists(tailOfLogPath)) { log.debug("{} already existed, skipping tail", tailOfLogPath); @@ -137,7 +146,7 @@ private void copyLogTail() { try { new SimpleProcessManager(log).runCommand(cmd, Redirect.to(tailOfLogPath.toFile())); } catch (Throwable t) { - log.error("Failed saving tail of log {} to {}", taskDefinition.getServiceLogOut(), configuration.getServiceFinishedTailLog(), t); + log.error("Failed saving tail of log {} to {}", taskDefinition.getServiceLogOut(), taskDefinition.getServiceFinishedTailLogPath(), t); } } @@ -218,7 +227,7 @@ private boolean writeTailMetadata(boolean finished) { private String getS3KeyPattern(String s3KeyPattern) { final SingularityTaskId singularityTaskId = getSingularityTaskId(); - return SingularityS3FormatHelper.getS3KeyFormat(s3KeyPattern, singularityTaskId, taskDefinition.getExecutorData().getLoggingTag()); + return SingularityS3FormatHelper.getS3KeyFormat(s3KeyPattern, singularityTaskId, taskDefinition.getExecutorData().getLoggingTag(), taskDefinition.getExecutorData().getRequestGroup().or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME)); } private SingularityTaskId getSingularityTaskId() { @@ -234,13 +243,16 @@ public Path getLogrotateCronPath() { } private boolean writeS3MetadataFile(String filenameHint, Path pathToS3Directory, String globForS3Files, Optional s3Bucket, Optional s3KeyPattern, boolean finished, - Optional s3StorageClass, Optional applyS3StorageClassAfterBytes) { - final String s3UploaderBucket = taskDefinition.getExecutorData().getLoggingS3Bucket().or(configuration.getS3UploaderBucket()); + Optional s3StorageClass, Optional applyS3StorageClassAfterBytes) { + final String s3UploaderBucket = s3Bucket.or(taskDefinition.getExecutorData().getDefaultS3Bucket()); + + if (Strings.isNullOrEmpty(s3UploaderBucket)) { + log.warn("No s3 bucket specified, not writing s3 metadata for file matcher {}", globForS3Files); + return false; + } - S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3Bucket.or(s3UploaderBucket), - getS3KeyPattern(s3KeyPattern.or(configuration.getS3UploaderKeyPattern())), finished, Optional. absent(), - Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), - s3StorageClass, applyS3StorageClassAfterBytes); + S3UploadMetadata s3UploadMetadata = new S3UploadMetadata(pathToS3Directory.toString(), globForS3Files, s3UploaderBucket, getS3KeyPattern(s3KeyPattern.or(taskDefinition.getExecutorData().getS3UploaderKeyPattern())), finished, Optional. absent(), + Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), s3StorageClass, applyS3StorageClassAfterBytes); String s3UploadMetadataFileName = String.format("%s-%s%s", taskDefinition.getTaskId(), filenameHint, baseConfiguration.getS3UploaderMetadataSuffix()); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java index 67f75d523c..29d856a6c0 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskProcessBuilder.java @@ -77,7 +77,7 @@ public ProcessBuilder call() throws Exception { task.getArtifactVerifier().checkSignatures(); - ProcessBuilder processBuilder = buildProcessBuilder(task.getTaskInfo(), executorData); + ProcessBuilder processBuilder = buildProcessBuilder(task.getTaskInfo(), executorData, task.getTaskDefinition().getServiceLogFileName()); task.getTaskLogManager().setup(); @@ -107,7 +107,7 @@ private String getExecutorUser() { return System.getProperty("user.name"); // TODO: better way to do this? } - private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData executorData) { + private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData executorData, String serviceLog) { final String cmd = getCommand(executorData); RunnerContext runnerContext = new RunnerContext( @@ -115,8 +115,8 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu configuration.getTaskAppDirectory(), configuration.getLogrotateToDirectory(), executorData.getUser().or(configuration.getDefaultRunAsUser()), - configuration.getServiceLog(), - serviceLogOutPath(), + serviceLog, + serviceLogOutPath(serviceLog), task.getTaskId(), executorData.getMaxTaskThreads().or(configuration.getMaxTaskThreads()), !getExecutorUser().equals(executorData.getUser().or(configuration.getDefaultRunAsUser())), @@ -151,10 +151,10 @@ private ProcessBuilder buildProcessBuilder(TaskInfo taskInfo, ExecutorData execu return processBuilder; } - private String serviceLogOutPath() { + private String serviceLogOutPath(String serviceLog) { Path basePath = task.getTaskDefinition().getTaskDirectoryPath(); Path app = basePath.resolve(configuration.getTaskAppDirectory()).normalize(); - return app.relativize(basePath).resolve(configuration.getServiceLog()).toString(); + return app.relativize(basePath).resolve(serviceLog).toString(); } @Override diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java index 232018e278..884f74110f 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/SingularityExecutorCleanup.java @@ -18,6 +18,7 @@ import org.slf4j.LoggerFactory; import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -29,6 +30,7 @@ import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularityTask; +import com.hubspot.singularity.SingularityTaskExecutorData; import com.hubspot.singularity.SingularityTaskHistory; import com.hubspot.singularity.SingularityTaskHistoryUpdate; import com.hubspot.singularity.client.SingularityClient; @@ -138,18 +140,20 @@ public SingularityExecutorCleanupStatistics clean() { statisticsBldr.incrTotalTaskFiles(); try { - Optional taskDefinition = jsonObjectFileHelper.read(file, LOG, SingularityExecutorTaskDefinition.class); + Optional maybeTaskDefinition = jsonObjectFileHelper.read(file, LOG, SingularityExecutorTaskDefinition.class); - if (!taskDefinition.isPresent()) { + if (!maybeTaskDefinition.isPresent()) { statisticsBldr.incrInvalidTasks(); continue; } - final String taskId = taskDefinition.get().getTaskId(); + SingularityExecutorTaskDefinition taskDefinition = withDefaults(maybeTaskDefinition.get()); + + final String taskId = taskDefinition.getTaskId(); LOG.info("{} - Starting possible cleanup", taskId); - if (runningTaskIds.contains(taskId) || executorStillRunning(taskDefinition.get())) { + if (runningTaskIds.contains(taskId) || executorStillRunning(taskDefinition)) { statisticsBldr.incrRunningTasksIgnored(); continue; } @@ -165,7 +169,7 @@ public SingularityExecutorCleanupStatistics clean() { continue; } - TaskCleanupResult result = cleanTask(taskDefinition.get(), taskHistory); + TaskCleanupResult result = cleanTask(taskDefinition, taskHistory); LOG.info("{} - {}", taskId, result); @@ -193,6 +197,32 @@ public SingularityExecutorCleanupStatistics clean() { return statisticsBldr.build(); } + private SingularityExecutorTaskDefinition withDefaults(SingularityExecutorTaskDefinition oldDefinition) { + return new SingularityExecutorTaskDefinition( + oldDefinition.getTaskId(), + new SingularityTaskExecutorData( + oldDefinition.getExecutorData(), + oldDefinition.getExecutorData().getS3UploaderAdditionalFiles() == null ? cleanupConfiguration.getS3UploaderAdditionalFiles() : oldDefinition.getExecutorData().getS3UploaderAdditionalFiles(), + Strings.isNullOrEmpty(oldDefinition.getExecutorData().getDefaultS3Bucket()) ? cleanupConfiguration.getDefaultS3Bucket() : oldDefinition.getExecutorData().getDefaultS3Bucket(), + Strings.isNullOrEmpty(oldDefinition.getExecutorData().getS3UploaderKeyPattern()) ? cleanupConfiguration.getS3KeyFormat(): oldDefinition.getExecutorData().getS3UploaderKeyPattern(), + Strings.isNullOrEmpty(oldDefinition.getExecutorData().getServiceLog()) ? cleanupConfiguration.getDefaultServiceLog() : oldDefinition.getExecutorData().getServiceLog(), + Strings.isNullOrEmpty(oldDefinition.getExecutorData().getServiceFinishedTailLog()) ? cleanupConfiguration.getDefaultServiceFinishedTailLog() : oldDefinition.getExecutorData().getServiceFinishedTailLog(), + oldDefinition.getExecutorData().getRequestGroup(), + oldDefinition.getExecutorData().getS3StorageClass(), + oldDefinition.getExecutorData().getApplyS3StorageClassAfterBytes() + ), + oldDefinition.getTaskDirectory(), + oldDefinition.getExecutorPid(), + Strings.isNullOrEmpty(oldDefinition.getServiceLogFileName()) ? cleanupConfiguration.getDefaultServiceLog() :oldDefinition.getServiceLogFileName(), + oldDefinition.getServiceLogOutExtension(), + Strings.isNullOrEmpty(oldDefinition.getServiceFinishedTailLogFileName()) ? cleanupConfiguration.getDefaultServiceFinishedTailLog() : oldDefinition.getServiceFinishedTailLogFileName(), + oldDefinition.getTaskAppDirectory(), + oldDefinition.getExecutorBashOut(), + oldDefinition.getLogrotateStateFile(), + oldDefinition.getSignatureVerifyOut() + ); + } + private boolean isDecommissioned() { Collection slaves = singularityClient.getSlaves(Optional.of(MachineState.DECOMMISSIONED)); boolean decommissioned = false; diff --git a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/config/SingularityExecutorCleanupConfiguration.java b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/config/SingularityExecutorCleanupConfiguration.java index 3ef555fc3c..45017173d0 100644 --- a/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/config/SingularityExecutorCleanupConfiguration.java +++ b/SingularityExecutorCleanup/src/main/java/com/hubspot/singularity/executor/cleanup/config/SingularityExecutorCleanupConfiguration.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; import com.hubspot.singularity.SingularityClientCredentials; +import com.hubspot.singularity.SingularityS3UploaderFile; import com.hubspot.singularity.runner.base.configuration.BaseRunnerConfiguration; import com.hubspot.singularity.runner.base.configuration.Configuration; import com.hubspot.singularity.runner.base.constraints.DirectoryExists; @@ -56,6 +57,25 @@ public class SingularityExecutorCleanupConfiguration extends BaseRunnerConfigura @JsonProperty private CompressionType compressionType = CompressionType.GZIP; + @NotEmpty + private String defaultServiceLog = "service.log"; + + @NotEmpty + private String defaultServiceFinishedTailLog = "tail_of_finished_service.log"; + + @NotNull + private List s3UploaderAdditionalFiles = Collections.singletonList(SingularityS3UploaderFile.fromString("service.log")); + + @NotNull + private String defaultS3Bucket = ""; + + /** + * S3 Key format for finding logs. Should be the same as + * configuration set for SingularityService + */ + @NotNull + private String s3KeyFormat = "%requestId/%Y/%m/%taskId_%index-%s-%filename"; + public SingularityExecutorCleanupConfiguration() { super(Optional.of("singularity-executor-cleanup.log")); } @@ -140,18 +160,69 @@ public void setCompressionType(CompressionType compressionType) { this.compressionType = compressionType; } + public String getDefaultServiceLog() { + return defaultServiceLog; + } + + public SingularityExecutorCleanupConfiguration setDefaultServiceLog(String defaultServiceLog) { + this.defaultServiceLog = defaultServiceLog; + return this; + } + + public String getDefaultServiceFinishedTailLog() { + return defaultServiceFinishedTailLog; + } + + public SingularityExecutorCleanupConfiguration setDefaultServiceFinishedTailLog(String defaultServiceFinishedTailLog) { + this.defaultServiceFinishedTailLog = defaultServiceFinishedTailLog; + return this; + } + + public List getS3UploaderAdditionalFiles() { + return s3UploaderAdditionalFiles; + } + + public SingularityExecutorCleanupConfiguration setS3UploaderAdditionalFiles(List s3UploaderAdditionalFiles) { + this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles; + return this; + } + + public String getDefaultS3Bucket() { + return defaultS3Bucket; + } + + public SingularityExecutorCleanupConfiguration setDefaultS3Bucket(String defaultS3Bucket) { + this.defaultS3Bucket = defaultS3Bucket; + return this; + } + + public String getS3KeyFormat() { + return s3KeyFormat; + } + + public SingularityExecutorCleanupConfiguration setS3KeyFormat(String s3KeyFormat) { + this.s3KeyFormat = s3KeyFormat; + return this; + } + @Override public String toString() { - return "SingularityExecutorCleanupConfiguration[" + - "safeModeWontRunWithNoTasks=" + safeModeWontRunWithNoTasks + - ", executorCleanupResultsDirectory='" + executorCleanupResultsDirectory + '\'' + - ", executorCleanupResultsSuffix='" + executorCleanupResultsSuffix + '\'' + - ", cleanupAppDirectoryOfFailedTasksAfterMillis=" + cleanupAppDirectoryOfFailedTasksAfterMillis + - ", singularityHosts=" + singularityHosts + - ", singularityContextPath='" + singularityContextPath + '\'' + - ", runDockerCleanup=" + runDockerCleanup + - ", singularityClientCredentials=" + singularityClientCredentials + - ", compressionType=" + compressionType + - ']'; + return "SingularityExecutorCleanupConfiguration{" + + "safeModeWontRunWithNoTasks=" + safeModeWontRunWithNoTasks + + ", executorCleanupResultsDirectory='" + executorCleanupResultsDirectory + '\'' + + ", executorCleanupResultsSuffix='" + executorCleanupResultsSuffix + '\'' + + ", cleanupAppDirectoryOfFailedTasksAfterMillis=" + cleanupAppDirectoryOfFailedTasksAfterMillis + + ", singularityHosts=" + singularityHosts + + ", singularityContextPath='" + singularityContextPath + '\'' + + ", runDockerCleanup=" + runDockerCleanup + + ", singularityClientCredentials=" + singularityClientCredentials + + ", cleanTasksWhenDecommissioned=" + cleanTasksWhenDecommissioned + + ", compressionType=" + compressionType + + ", defaultServiceLog='" + defaultServiceLog + '\'' + + ", defaultServiceFinishedTailLog='" + defaultServiceFinishedTailLog + '\'' + + ", s3UploaderAdditionalFiles=" + s3UploaderAdditionalFiles + + ", defaultS3Bucket='" + defaultS3Bucket + '\'' + + ", s3KeyFormat='" + s3KeyFormat + '\'' + + "} " + super.toString(); } } diff --git a/SingularityS3Base/pom.xml b/SingularityS3Base/pom.xml index ef83a762e7..23a67ee9ad 100644 --- a/SingularityS3Base/pom.xml +++ b/SingularityS3Base/pom.xml @@ -31,13 +31,23 @@ slf4j-api - + org.slf4j jcl-over-slf4j runtime + + com.amazonaws + aws-java-sdk-core + + + + com.amazonaws + aws-java-sdk-s3 + + ch.qos.logback logback-classic @@ -49,11 +59,6 @@ jackson-annotations - - net.java.dev.jets3t - jets3t - - javax.validation validation-api diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactChunkDownloader.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactChunkDownloader.java index 1bef8733b6..5f51cb7b0d 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactChunkDownloader.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactChunkDownloader.java @@ -12,10 +12,11 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.jets3t.service.S3Service; -import org.jets3t.service.model.S3Object; import org.slf4j.Logger; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.S3Object; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.hubspot.deploy.S3Artifact; @@ -26,7 +27,7 @@ public class S3ArtifactChunkDownloader implements Callable { private final SingularityS3Configuration configuration; - private final S3Service s3; + private final AmazonS3 s3; private final S3Artifact s3Artifact; private final Path downloadTo; private final int chunk; @@ -37,7 +38,7 @@ public class S3ArtifactChunkDownloader implements Callable { private int retryNum; - public S3ArtifactChunkDownloader(SingularityS3Configuration configuration, Logger log, S3Service s3, S3Artifact s3Artifact, Path downloadTo, int chunk, long chunkSize, long length, SingularityRunnerExceptionNotifier exceptionNotifier) { + public S3ArtifactChunkDownloader(SingularityS3Configuration configuration, Logger log, AmazonS3 s3, S3Artifact s3Artifact, Path downloadTo, int chunk, long chunkSize, long length, SingularityRunnerExceptionNotifier exceptionNotifier) { this.configuration = configuration; this.log = log; this.s3 = s3; @@ -99,9 +100,12 @@ public Path call() throws Exception { log.info("Downloading {} - chunk {} (retry {}) ({}-{}) to {}", s3Artifact.getFilename(), chunk, retryNum, byteRangeStart, byteRangeEnd, chunkPath); - S3Object fetchedObject = s3.getObject(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey(), null, null, null, null, byteRangeStart, byteRangeEnd); + GetObjectRequest getObjectRequest = new GetObjectRequest(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey()) + .withRange(byteRangeStart, byteRangeEnd); - try (InputStream is = fetchedObject.getDataInputStream()) { + S3Object fetchedObject = s3.getObject(getObjectRequest); + + try (InputStream is = fetchedObject.getObjectContent()) { Files.copy(is, chunkPath, StandardCopyOption.REPLACE_EXISTING); } diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactDownloader.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactDownloader.java index 455289bdcc..59180c329f 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactDownloader.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/S3ArtifactDownloader.java @@ -13,14 +13,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import org.jets3t.service.Constants; -import org.jets3t.service.Jets3tProperties; -import org.jets3t.service.S3Service; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.StorageObject; -import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.S3Object; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; @@ -57,32 +56,31 @@ public void download(S3Artifact s3Artifact, Path downloadTo) { } } - private AWSCredentials getCredentialsForBucket(String bucketName) { + private BasicAWSCredentials getCredentialsForBucket(String bucketName) { if (configuration.getS3BucketCredentials().containsKey(bucketName)) { return configuration.getS3BucketCredentials().get(bucketName).toAWSCredentials(); } - return new AWSCredentials(configuration.getS3AccessKey().get(), configuration.getS3SecretKey().get()); + return new BasicAWSCredentials(configuration.getS3AccessKey().get(), configuration.getS3SecretKey().get()); } private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) throws Exception { log.info("Downloading {}", s3Artifact); - Jets3tProperties jets3tProperties = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME); - jets3tProperties.setProperty("httpclient.socket-timeout-ms", Long.toString(configuration.getS3ChunkDownloadTimeoutMillis())); - - final S3Service s3 = new RestS3Service(getCredentialsForBucket(s3Artifact.getS3Bucket()), null, null, jets3tProperties); + ClientConfiguration clientConfiguration = new ClientConfiguration() + .withSocketTimeout(configuration.getS3ChunkDownloadTimeoutMillis()); + final AmazonS3 s3Client = new AmazonS3Client(getCredentialsForBucket(s3Artifact.getS3Bucket()), clientConfiguration); long length = 0; if (s3Artifact.getFilesize().isPresent()) { length = s3Artifact.getFilesize().get(); } else { - StorageObject details = s3.getObjectDetails(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey()); + S3Object details = s3Client.getObject(s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey()); Preconditions.checkNotNull(details, "Couldn't find object at %s/%s", s3Artifact.getS3Bucket(), s3Artifact.getS3ObjectKey()); - length = details.getContentLength(); + length = details.getObjectMetadata().getContentLength(); } int numChunks = (int) (length / configuration.getS3ChunkSize()); @@ -99,7 +97,7 @@ private void downloadThrows(final S3Artifact s3Artifact, final Path downloadTo) final List> futures = Lists.newArrayListWithCapacity(numChunks); for (int chunk = 0; chunk < numChunks; chunk++) { - futures.add(chunkExecutorService.submit(new S3ArtifactChunkDownloader(configuration, log, s3, s3Artifact, downloadTo, chunk, chunkSize, length, exceptionNotifier))); + futures.add(chunkExecutorService.submit(new S3ArtifactChunkDownloader(configuration, log, s3Client, s3Artifact, downloadTo, chunk, chunkSize, length, exceptionNotifier))); } long remainingMillis = configuration.getS3DownloadTimeoutMillis(); diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Configuration.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Configuration.java index 1d5e38edc4..e7d3da1aaf 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Configuration.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Configuration.java @@ -1,7 +1,5 @@ package com.hubspot.singularity.s3.base.config; -import static com.hubspot.mesos.JavaUtils.obfuscateValue; - import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -45,7 +43,7 @@ public class SingularityS3Configuration extends BaseRunnerConfiguration { @Min(1) @JsonProperty - private long s3ChunkDownloadTimeoutMillis = TimeUnit.SECONDS.toMillis(30); + private int s3ChunkDownloadTimeoutMillis = (int) TimeUnit.SECONDS.toMillis(30); @Min(1) @JsonProperty @@ -131,11 +129,11 @@ public void setLocalDownloadPath(String localDownloadPath) { this.localDownloadPath = localDownloadPath; } - public long getS3ChunkDownloadTimeoutMillis() { + public int getS3ChunkDownloadTimeoutMillis() { return s3ChunkDownloadTimeoutMillis; } - public void setS3ChunkDownloadTimeoutMillis(long s3ChunkDownloadTimeoutMillis) { + public void setS3ChunkDownloadTimeoutMillis(int s3ChunkDownloadTimeoutMillis) { this.s3ChunkDownloadTimeoutMillis = s3ChunkDownloadTimeoutMillis; } @@ -149,17 +147,17 @@ public void setS3BucketCredentials(Map s3Bucke @Override public String toString() { - return "SingularityS3Configuration[" + - "artifactCacheDirectory='" + artifactCacheDirectory + '\'' + - ", s3AccessKey='" + obfuscateValue(s3AccessKey) + '\'' + - ", s3SecretKey='" + obfuscateValue(s3SecretKey) + '\'' + - ", s3ChunkSize=" + s3ChunkSize + - ", s3DownloadTimeoutMillis=" + s3DownloadTimeoutMillis + - ", s3ChunkDownloadTimeoutMillis=" + s3ChunkDownloadTimeoutMillis + - ", s3ChunkRetries=" + s3ChunkRetries + - ", localDownloadHttpPort=" + localDownloadHttpPort + - ", localDownloadPath='" + localDownloadPath + '\'' + - ", s3BucketCredentials=" + s3BucketCredentials + - ']'; + return "SingularityS3Configuration{" + + "artifactCacheDirectory='" + artifactCacheDirectory + '\'' + + ", s3AccessKey=" + s3AccessKey + + ", s3SecretKey=" + s3SecretKey + + ", s3ChunkSize=" + s3ChunkSize + + ", s3DownloadTimeoutMillis=" + s3DownloadTimeoutMillis + + ", s3ChunkDownloadTimeoutMillis=" + s3ChunkDownloadTimeoutMillis + + ", s3ChunkRetries=" + s3ChunkRetries + + ", localDownloadHttpPort=" + localDownloadHttpPort + + ", localDownloadPath='" + localDownloadPath + '\'' + + ", s3BucketCredentials=" + s3BucketCredentials + + "} " + super.toString(); } } diff --git a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Credentials.java b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Credentials.java index 8880337c26..5a1d291210 100644 --- a/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Credentials.java +++ b/SingularityS3Base/src/main/java/com/hubspot/singularity/s3/base/config/SingularityS3Credentials.java @@ -4,8 +4,7 @@ import java.util.Objects; -import org.jets3t.service.security.AWSCredentials; - +import com.amazonaws.auth.BasicAWSCredentials; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -59,7 +58,7 @@ public String toString() { } @JsonIgnore - public AWSCredentials toAWSCredentials() { - return new AWSCredentials(accessKey, secretKey); + public BasicAWSCredentials toAWSCredentials() { + return new BasicAWSCredentials(accessKey, secretKey); } } diff --git a/SingularityS3Uploader/pom.xml b/SingularityS3Uploader/pom.xml index 9f68c490e8..d0e6409b87 100644 --- a/SingularityS3Uploader/pom.xml +++ b/SingularityS3Uploader/pom.xml @@ -50,7 +50,17 @@ slf4j-api - + + com.amazonaws + aws-java-sdk-s3 + + + + com.amazonaws + aws-java-sdk-core + + + org.slf4j jcl-over-slf4j @@ -68,11 +78,6 @@ metrics-core - - net.java.dev.jets3t - jets3t - - javax.validation validation-api diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java index e04ec25ea1..a436826e0c 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3Uploader.java @@ -1,6 +1,6 @@ package com.hubspot.singularity.s3uploader; -import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.charset.Charset; @@ -11,25 +11,29 @@ import java.nio.file.PathMatcher; import java.nio.file.Paths; import java.nio.file.attribute.UserDefinedFileAttributeView; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Bucket; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.model.StorageObject; -import org.jets3t.service.security.AWSCredentials; -import org.jets3t.service.utils.MultipartUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.AbortMultipartUploadRequest; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; +import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PartETag; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.model.StorageClass; +import com.amazonaws.services.s3.model.UploadPartRequest; import com.codahale.metrics.Timer.Context; import com.github.rholder.retry.RetryException; import com.github.rholder.retry.Retryer; @@ -44,15 +48,15 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.hubspot.mesos.JavaUtils; -import com.hubspot.singularity.SingularityS3Log; import com.hubspot.singularity.SingularityS3FormatHelper; +import com.hubspot.singularity.SingularityS3Log; import com.hubspot.singularity.runner.base.sentry.SingularityRunnerExceptionNotifier; import com.hubspot.singularity.runner.base.shared.S3UploadMetadata; import com.hubspot.singularity.runner.base.shared.SimpleProcessManager; import com.hubspot.singularity.s3uploader.config.SingularityS3UploaderConfiguration; import com.hubspot.singularity.s3uploader.config.SingularityS3UploaderContentHeaders; -public class SingularityS3Uploader implements Closeable { +public class SingularityS3Uploader { private static final Logger LOG = LoggerFactory.getLogger(SingularityS3Uploader.class); private static final String LOG_START_TIME_ATTR = "logstart"; @@ -63,8 +67,8 @@ public class SingularityS3Uploader implements Closeable { private final PathMatcher pathMatcher; private final Optional finishedPathMatcher; private final String fileDirectory; - private final S3Service s3Service; - private final S3Bucket s3Bucket; + private final AmazonS3 s3Client; + private final String s3BucketName; private final Path metadataPath; private final SingularityS3UploaderMetrics metrics; private final String logIdentifier; @@ -72,20 +76,15 @@ public class SingularityS3Uploader implements Closeable { private final SingularityS3UploaderConfiguration configuration; private final SingularityRunnerExceptionNotifier exceptionNotifier; - public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath, - SingularityS3UploaderConfiguration configuration, String hostname, SingularityRunnerExceptionNotifier exceptionNotifier) { - AWSCredentials credentials = defaultCredentials; + public SingularityS3Uploader(BasicAWSCredentials defaultCredentials, S3UploadMetadata uploadMetadata, FileSystem fileSystem, SingularityS3UploaderMetrics metrics, Path metadataPath, + SingularityS3UploaderConfiguration configuration, String hostname, SingularityRunnerExceptionNotifier exceptionNotifier) { + BasicAWSCredentials credentials = defaultCredentials; if (uploadMetadata.getS3SecretKey().isPresent() && uploadMetadata.getS3AccessKey().isPresent()) { - credentials = new AWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3SecretKey().get()); - } - - try { - this.s3Service = new RestS3Service(credentials); - } catch (S3ServiceException e) { - throw Throwables.propagate(e); + credentials = new BasicAWSCredentials(uploadMetadata.getS3AccessKey().get(), uploadMetadata.getS3SecretKey().get()); } + this.s3Client = new AmazonS3Client(credentials); this.metrics = metrics; this.uploadMetadata = uploadMetadata; this.fileDirectory = uploadMetadata.getDirectory(); @@ -98,7 +97,7 @@ public SingularityS3Uploader(AWSCredentials defaultCredentials, S3UploadMetadata } this.hostname = hostname; - this.s3Bucket = new S3Bucket(uploadMetadata.getS3Bucket()); + this.s3BucketName = uploadMetadata.getS3Bucket(); this.metadataPath = metadataPath; this.logIdentifier = String.format("[%s]", metadataPath.getFileName()); this.configuration = configuration; @@ -113,15 +112,6 @@ public S3UploadMetadata getUploadMetadata() { return uploadMetadata; } - @Override - public void close() throws IOException { - try { - s3Service.shutdown(); - } catch (ServiceException e) { - throw new IOException(e); - } - } - @Override public String toString() { return "SingularityS3Uploader [uploadMetadata=" + uploadMetadata + ", metadataPath=" + metadataPath + "]"; @@ -186,10 +176,10 @@ private void uploadBatch(List toUpload) { metrics.upload(); success++; Files.delete(file); - } catch (S3ServiceException se) { + } catch (AmazonS3Exception se) { metrics.error(); - LOG.warn("{} Couldn't upload {} due to {} ({}) - {}", logIdentifier, file, se.getErrorCode(), se.getResponseCode(), se.getErrorMessage(), se); - exceptionNotifier.notify(String.format("S3ServiceException during upload (%s)", se.getMessage()), se, ImmutableMap.of("logIdentifier", logIdentifier, "file", file.toString(), "errorCode", se.getErrorCode(), "responseCode", Integer.toString(se.getResponseCode()), "errorMessage", se.getErrorMessage())); + LOG.warn("{} Couldn't upload {} due to {} - {}", logIdentifier, file, se.getErrorCode(), se.getErrorMessage(), se); + exceptionNotifier.notify(String.format("S3ServiceException during upload (%s)", se.getMessage()), se, ImmutableMap.of("logIdentifier", logIdentifier, "file", file.toString(), "errorCode", se.getErrorCode(), "errorMessage", se.getErrorMessage())); } catch (RetryException re) { metrics.error(); LOG.warn("{} Couldn't upload or delete {}", logIdentifier, file, re); @@ -213,7 +203,7 @@ private void uploadSingle(int sequence, Path file) throws Exception { Callable uploader = new Uploader(sequence, file); Retryer retryer = RetryerBuilder.newBuilder() - .retryIfExceptionOfType(S3ServiceException.class) + .retryIfExceptionOfType(AmazonS3Exception.class) .retryIfRuntimeException() .withWaitStrategy(WaitStrategies.fixedWait(configuration.getRetryWaitMs(), TimeUnit.MILLISECONDS)) .withStopStrategy(StopStrategies.stopAfterAttempt(configuration.getRetryCount())) @@ -249,11 +239,10 @@ public Boolean call() throws Exception { final String key = SingularityS3FormatHelper.getKey(uploadMetadata.getS3KeyFormat(), sequence, Files.getLastModifiedTime(file).toMillis(), Objects.toString(file.getFileName()), hostname); long fileSizeBytes = Files.size(file); - LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, s3Bucket.getName(), key, fileSizeBytes); + LOG.info("{} Uploading {} to {}/{} (size {})", logIdentifier, file, s3BucketName, key, fileSizeBytes); try { - S3Object object = new S3Object(s3Bucket, file.toFile()); - object.setKey(key); + ObjectMetadata objectMetadata = new ObjectMetadata(); Set supportedViews = FileSystems.getDefault().supportedFileAttributeViews(); LOG.trace("Supported attribute views are {}", supportedViews); @@ -264,12 +253,12 @@ public Boolean call() throws Exception { LOG.debug("Found file attributes {} for file {}", attributes, file); Optional maybeStartTime = readFileAttributeAsLong(LOG_START_TIME_ATTR, view, attributes); if (maybeStartTime.isPresent()) { - object.addMetadata(SingularityS3Log.LOG_START_S3_ATTR, maybeStartTime.get().toString()); + objectMetadata.addUserMetadata(SingularityS3Log.LOG_START_S3_ATTR, maybeStartTime.get().toString()); LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_START_S3_ATTR, maybeStartTime.get()); } Optional maybeEndTime = readFileAttributeAsLong(LOG_END_TIME_ATTR, view, attributes); if (maybeEndTime.isPresent()) { - object.addMetadata(SingularityS3Log.LOG_END_S3_ATTR, maybeEndTime.get().toString()); + objectMetadata.addUserMetadata(SingularityS3Log.LOG_END_S3_ATTR, maybeEndTime.get().toString()); LOG.debug("Added extra metadata for object ({}:{})", SingularityS3Log.LOG_END_S3_ATTR, maybeEndTime.get()); } } catch (Exception e) { @@ -281,26 +270,32 @@ public Boolean call() throws Exception { if (file.toString().endsWith(contentHeaders.getFilenameEndsWith())) { LOG.debug("{} Using content headers {} for file {}", logIdentifier, contentHeaders, file); if (contentHeaders.getContentType().isPresent()) { - object.setContentType(contentHeaders.getContentType().get()); + objectMetadata.setContentType(contentHeaders.getContentType().get()); } if (contentHeaders.getContentEncoding().isPresent()) { - object.setContentEncoding(contentHeaders.getContentEncoding().get()); + objectMetadata.setContentEncoding(contentHeaders.getContentEncoding().get()); } break; } } + Optional maybeStorageClass = Optional.absent(); + if (shouldApplyStorageClass(fileSizeBytes)) { LOG.debug("{} adding storage class {} to {}", logIdentifier, uploadMetadata.getS3StorageClass().get(), file); - object.addMetadata("x-amz-storage-class", uploadMetadata.getS3StorageClass().get()); + maybeStorageClass = Optional.of(StorageClass.fromValue(uploadMetadata.getS3StorageClass().get())); } - LOG.debug("Uploading object with metadata {}", object.getMetadataMap()); + LOG.debug("Uploading object with metadata {}", objectMetadata); if (fileSizeBytes > configuration.getMaxSingleUploadSizeBytes()) { - multipartUpload(object); + multipartUpload(key, file.toFile(), objectMetadata, maybeStorageClass); } else { - s3Service.putObject(s3Bucket, object); + PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, key, file.toFile()).withMetadata(objectMetadata); + if (maybeStorageClass.isPresent()) { + putObjectRequest.setStorageClass(maybeStorageClass.get()); + } + s3Client.putObject(putObjectRequest); } } catch (Exception e) { LOG.warn("Exception uploading {}", file, e); @@ -352,12 +347,39 @@ public static boolean fileOpen(Path path) { return false; } - private void multipartUpload(S3Object object) throws Exception { + private void multipartUpload(String key, File file, ObjectMetadata objectMetadata, Optional maybeStorageClass) throws Exception { + List partETags = new ArrayList<>(); + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(s3BucketName, key, objectMetadata); + if (maybeStorageClass.isPresent()) { + initRequest.setStorageClass(maybeStorageClass.get()); + } + InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest); - List objectsToUploadAsMultipart = Arrays.asList(object); + long contentLength = file.length(); + long partSize = configuration.getUploadPartSize(); - MultipartUtils mpUtils = new MultipartUtils(configuration.getUploadPartSize()); - mpUtils.uploadObjects(s3Bucket.getName(), s3Service, objectsToUploadAsMultipart, null); + try { + long filePosition = 0; + for (int i = 1; filePosition < contentLength; i++) { + partSize = Math.min(partSize, (contentLength - filePosition)); + UploadPartRequest uploadRequest = new UploadPartRequest() + .withBucketName(s3BucketName) + .withKey(key) + .withUploadId(initResponse.getUploadId()) + .withPartNumber(i) + .withFileOffset(filePosition) + .withFile(file) + .withPartSize(partSize); + partETags.add(s3Client.uploadPart(uploadRequest).getPartETag()); + filePosition += partSize; + } + + CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(s3BucketName, key, initResponse.getUploadId(), partETags); + s3Client.completeMultipartUpload(completeRequest); + } catch (Exception e) { + s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(s3BucketName, key, initResponse.getUploadId())); + Throwables.propagate(e); + } } } diff --git a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java index 2916283785..4bc587d344 100644 --- a/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java +++ b/SingularityS3Uploader/src/main/java/com/hubspot/singularity/s3uploader/SingularityS3UploaderDriver.java @@ -24,10 +24,10 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.jets3t.service.security.AWSCredentials; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.auth.BasicAWSCredentials; import com.google.common.base.Optional; import com.google.common.base.Predicate; import com.google.common.base.Throwables; @@ -36,7 +36,6 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.google.common.io.Closeables; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.google.inject.Inject; import com.google.inject.name.Named; @@ -271,12 +270,10 @@ public Integer call() { expiring.remove(expiredUploader); try { - Closeables.close(expiredUploader, true); - LOG.debug("Deleting expired uploader {}", expiredUploader.getMetadataPath()); Files.delete(expiredUploader.getMetadataPath()); } catch (NoSuchFileException nfe) { - LOG.warn("File {} was alrady deleted"); + LOG.warn("File {} was already deleted", nfe.getFile()); } catch (IOException e) { LOG.warn("Couldn't delete {}", expiredUploader.getMetadataPath(), e); exceptionNotifier.notify("Could not delete metadata file", e, ImmutableMap.of("metadataPath", expiredUploader.getMetadataPath().toString())); @@ -358,13 +355,13 @@ private boolean handleNewOrModifiedS3Metadata(Path filename) throws IOException try { metrics.getUploaderCounter().inc(); - Optional bucketCreds = Optional.absent(); + Optional bucketCreds = Optional.absent(); if (configuration.getS3BucketCredentials().containsKey(metadata.getS3Bucket())) { bucketCreds = Optional.of(configuration.getS3BucketCredentials().get(metadata.getS3Bucket()).toAWSCredentials()); } - final AWSCredentials defaultCredentials = new AWSCredentials(configuration.getS3AccessKey().or(s3Configuration.getS3AccessKey()).get(), configuration.getS3SecretKey().or(s3Configuration.getS3SecretKey()).get()); + final BasicAWSCredentials defaultCredentials = new BasicAWSCredentials(configuration.getS3AccessKey().or(s3Configuration.getS3AccessKey()).get(), configuration.getS3SecretKey().or(s3Configuration.getS3SecretKey()).get()); SingularityS3Uploader uploader = new SingularityS3Uploader(bucketCreds.or(defaultCredentials), metadata, fileSystem, metrics, filename, configuration, hostname, exceptionNotifier); diff --git a/SingularityService/pom.xml b/SingularityService/pom.xml index ef1c4a558c..a8b1d6ffc1 100644 --- a/SingularityService/pom.xml +++ b/SingularityService/pom.xml @@ -102,7 +102,7 @@ slf4j-api - + org.slf4j jcl-over-slf4j @@ -328,8 +328,13 @@ - net.java.dev.jets3t - jets3t + com.amazonaws + aws-java-sdk-s3 + + + + com.amazonaws + aws-java-sdk-core diff --git a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java index c3a9b8a93d..c0dfbe79cf 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/SingularityMainModule.java @@ -6,7 +6,6 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.Collections; import java.util.Map; import java.util.UUID; import java.util.concurrent.ScheduledExecutorService; @@ -19,16 +18,14 @@ import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.LeaderLatchListener; import org.apache.curator.framework.state.ConnectionStateListener; -import org.jets3t.service.S3Service; -import org.jets3t.service.S3ServiceException; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.security.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3Client; import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Optional; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; import com.google.common.net.HostAndPort; import com.google.inject.Binder; import com.google.inject.Module; @@ -44,7 +41,7 @@ import com.hubspot.singularity.config.HistoryPurgingConfiguration; import com.hubspot.singularity.config.MesosConfiguration; import com.hubspot.singularity.config.S3Configuration; -import com.hubspot.singularity.config.S3GroupOverrideConfiguration; +import com.hubspot.singularity.config.S3GroupConfiguration; import com.hubspot.singularity.config.SMTPConfiguration; import com.hubspot.singularity.config.SentryConfiguration; import com.hubspot.singularity.config.SingularityConfiguration; @@ -53,6 +50,8 @@ import com.hubspot.singularity.config.ZooKeeperConfiguration; import com.hubspot.singularity.guice.DropwizardMetricRegistryProvider; import com.hubspot.singularity.guice.DropwizardObjectMapperProvider; +import com.hubspot.singularity.helpers.SingularityS3Service; +import com.hubspot.singularity.helpers.SingularityS3Services; import com.hubspot.singularity.hooks.LoadBalancerClient; import com.hubspot.singularity.hooks.LoadBalancerClientImpl; import com.hubspot.singularity.hooks.SingularityWebhookPoller; @@ -242,28 +241,21 @@ public SingularityTaskMetadataConfiguration taskMetadataConfiguration(Singularit @Provides @Singleton - public Optional s3Service(Optional config) throws S3ServiceException { - if (!config.isPresent()) { - return Optional.absent(); - } - - return Optional.of(new RestS3Service(new AWSCredentials(config.get().getS3AccessKey(), config.get().getS3SecretKey()))); - } - - @Provides - @Singleton - public Map s3ServiceGroupOverrides(Optional config) throws S3ServiceException { + public SingularityS3Services provideS3Services(Optional config) { if (!config.isPresent() || config.get().getGroupOverrides().isEmpty()) { - return Collections.emptyMap(); + return new SingularityS3Services(); } - final ImmutableMap.Builder s3ServiceBuilder = ImmutableMap.builder(); - - for (Map.Entry entry : config.get().getGroupOverrides().entrySet()) { - s3ServiceBuilder.put(entry.getKey(), new RestS3Service(new AWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey()))); + final ImmutableList.Builder s3ServiceBuilder = ImmutableList.builder(); + for (Map.Entry entry : config.get().getGroupOverrides().entrySet()) { + s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey())))); + } + for (Map.Entry entry : config.get().getGroupS3SearchConfigs().entrySet()) { + s3ServiceBuilder.add(new SingularityS3Service(entry.getKey(), entry.getValue().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(entry.getValue().getS3AccessKey(), entry.getValue().getS3SecretKey())))); } + SingularityS3Service defaultService = new SingularityS3Service(SingularityS3FormatHelper.DEFAULT_GROUP_NAME, config.get().getS3Bucket(), new AmazonS3Client(new BasicAWSCredentials(config.get().getS3AccessKey(), config.get().getS3SecretKey()))); - return s3ServiceBuilder.build(); + return new SingularityS3Services(s3ServiceBuilder.build(), defaultService); } @Provides diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java index 51ab6e028a..2d41b99cb1 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/CustomExecutorConfiguration.java @@ -2,6 +2,8 @@ import javax.validation.constraints.Min; +import org.hibernate.validator.constraints.NotEmpty; + public class CustomExecutorConfiguration { @Min(0) private double numCpus = 0; @@ -12,6 +14,12 @@ public class CustomExecutorConfiguration { @Min(0) private int diskMb = 0; + @NotEmpty + private String serviceLog = "service.log"; + + @NotEmpty + private String serviceFinishedTailLog = "tail_of_finished_service.log"; + public double getNumCpus() { return numCpus; } @@ -35,4 +43,20 @@ public int getDiskMb() { public void setDiskMb(int diskMb) { this.diskMb = diskMb; } + + public String getServiceLog() { + return serviceLog; + } + + public void setServiceLog(String serviceLog) { + this.serviceLog = serviceLog; + } + + public String getServiceFinishedTailLog() { + return serviceFinishedTailLog; + } + + public void setServiceFinishedTailLog(String serviceFinishedTailLog) { + this.serviceFinishedTailLog = serviceFinishedTailLog; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java index 8925aaf824..97d1a81dec 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/S3Configuration.java @@ -1,11 +1,16 @@ package com.hubspot.singularity.config; +import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import javax.validation.constraints.NotNull; +import com.google.common.base.Optional; +import com.hubspot.singularity.SingularityS3UploaderFile; + public class S3Configuration { @NotNull @@ -31,15 +36,21 @@ public class S3Configuration { private String s3Bucket; @NotNull - private Map groupOverrides = new HashMap<>(); + private Map groupOverrides = new HashMap<>(); + + /** + * When searching s3 for requests in these groups, additionally search these buckets + * for logs. Do not upload new logs to them (that is what `groupOverrides` is for) + */ + @NotNull + private Map groupS3SearchConfigs = new HashMap<>(); /** * S3 Key format for finding logs. Should be the same as - * configuration set for SingularityS3Uploader - * (e.g. '%requestId/%Y/%m/%taskId_%index-%s%fileext') + * configuration set for SingularityExecutorCleanup */ @NotNull - private String s3KeyFormat; + private String s3KeyFormat = "%requestId/%Y/%m/%taskId_%index-%s-%filename"; @NotNull private String s3AccessKey; @@ -47,6 +58,15 @@ public class S3Configuration { @NotNull private String s3SecretKey; + @NotNull + private List s3UploaderAdditionalFiles = Collections.singletonList(SingularityS3UploaderFile.fromString("service.log")); + + @NotNull + private Optional s3StorageClass = Optional.absent(); + + @NotNull + private Optional applyS3StorageClassAfterBytes = Optional.absent(); + public int getMaxS3Threads() { return maxS3Threads; } @@ -119,11 +139,44 @@ public void setS3SecretKey(String s3SecretKey) { this.s3SecretKey = s3SecretKey; } - public Map getGroupOverrides() { + public Map getGroupOverrides() { return groupOverrides; } - public void setGroupOverrides(Map groupOverrides) { + public void setGroupOverrides(Map groupOverrides) { this.groupOverrides = groupOverrides; } + + public Map getGroupS3SearchConfigs() { + return groupS3SearchConfigs; + } + + public S3Configuration setGroupS3SearchConfigs(Map groupS3SearchConfigs) { + this.groupS3SearchConfigs = groupS3SearchConfigs; + return this; + } + + public List getS3UploaderAdditionalFiles() { + return s3UploaderAdditionalFiles; + } + + public void setS3UploaderAdditionalFiles(List s3UploaderAdditionalFiles) { + this.s3UploaderAdditionalFiles = s3UploaderAdditionalFiles; + } + + public Optional getS3StorageClass() { + return s3StorageClass; + } + + public void setS3StorageClass(Optional s3StorageClass) { + this.s3StorageClass = s3StorageClass; + } + + public Optional getApplyS3StorageClassAfterBytes() { + return applyS3StorageClassAfterBytes; + } + + public void setApplyS3StorageClassAfterBytes(Optional applyS3StorageClassAfterBytes) { + this.applyS3StorageClassAfterBytes = applyS3StorageClassAfterBytes; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupOverrideConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupConfiguration.java similarity index 90% rename from SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupOverrideConfiguration.java rename to SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupConfiguration.java index a3cead8f35..79244ef09e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupOverrideConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/S3GroupConfiguration.java @@ -4,7 +4,7 @@ import javax.validation.constraints.NotNull; -public class S3GroupOverrideConfiguration { +public class S3GroupConfiguration { @NotNull private String s3Bucket; @@ -46,7 +46,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { return false; } - S3GroupOverrideConfiguration that = (S3GroupOverrideConfiguration) o; + S3GroupConfiguration that = (S3GroupConfiguration) o; return Objects.equals(s3Bucket, that.s3Bucket) && Objects.equals(s3AccessKey, that.s3AccessKey) && Objects.equals(s3SecretKey, that.s3SecretKey); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/helpers/S3ObjectSummaryHolder.java b/SingularityService/src/main/java/com/hubspot/singularity/helpers/S3ObjectSummaryHolder.java new file mode 100644 index 0000000000..4a6d85c536 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/helpers/S3ObjectSummaryHolder.java @@ -0,0 +1,53 @@ +package com.hubspot.singularity.helpers; + +import com.amazonaws.services.s3.model.S3ObjectSummary; + +public class S3ObjectSummaryHolder { + private final String group; + private final S3ObjectSummary objectSummary; + + public S3ObjectSummaryHolder(String group, S3ObjectSummary objectSummary) { + this.group = group; + this.objectSummary = objectSummary; + } + + public String getGroup() { + return group; + } + + public S3ObjectSummary getObjectSummary() { + return objectSummary; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + S3ObjectSummaryHolder that = (S3ObjectSummaryHolder) o; + + if (group != null ? !group.equals(that.group) : that.group != null) { + return false; + } + return objectSummary != null ? objectSummary.equals(that.objectSummary) : that.objectSummary == null; + } + + @Override + public int hashCode() { + int result = group != null ? group.hashCode() : 0; + result = 31 * result + (objectSummary != null ? objectSummary.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "S3ObjectSummaryHolder{" + + "group='" + group + '\'' + + ", objectSummary=" + objectSummary + + '}'; + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Service.java b/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Service.java new file mode 100644 index 0000000000..440089bd0d --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Service.java @@ -0,0 +1,36 @@ +package com.hubspot.singularity.helpers; + +import com.amazonaws.services.s3.AmazonS3; + +public class SingularityS3Service { + private final String group; + private final String bucket; + private final AmazonS3 s3Client; + + public SingularityS3Service(String group, String bucket, AmazonS3 s3Client) { + this.group = group; + this.bucket = bucket; + this.s3Client = s3Client; + } + + public String getGroup() { + return group; + } + + public String getBucket() { + return bucket; + } + + public AmazonS3 getS3Client() { + return s3Client; + } + + @Override + public String toString() { + return "SingularityS3Service{" + + "group='" + group + '\'' + + ", bucket='" + bucket + '\'' + + ", s3Client=" + s3Client + + '}'; + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Services.java b/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Services.java new file mode 100644 index 0000000000..b0c9f36a31 --- /dev/null +++ b/SingularityService/src/main/java/com/hubspot/singularity/helpers/SingularityS3Services.java @@ -0,0 +1,43 @@ +package com.hubspot.singularity.helpers; + +import java.util.Collections; +import java.util.List; + +public class SingularityS3Services { + private final boolean s3ConfigPresent; + private final List s3Services; + private final SingularityS3Service defaultS3Service; + + public SingularityS3Services() { + this.s3ConfigPresent = false; + this.s3Services = Collections.emptyList(); + this.defaultS3Service = null; + } + + public SingularityS3Services(List s3Services, SingularityS3Service defaultS3Service) { + this.s3ConfigPresent = true; + this.s3Services = s3Services; + this.defaultS3Service = defaultS3Service; + } + + public boolean isS3ConfigPresent() { + return s3ConfigPresent; + } + + public List getS3Services() { + return s3Services; + } + + public SingularityS3Service getDefaultS3Service() { + return defaultS3Service; + } + + public SingularityS3Service getServiceByGroupAndBucketOrDefault(String group, String bucket) { + for (SingularityS3Service s3Service : s3Services) { + if (s3Service.getGroup().equals(group) && s3Service.getBucket().equals(bucket)) { + return s3Service; + } + } + return defaultS3Service; + } +} diff --git a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java index fa70323267..25360c8941 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilder.java @@ -36,7 +36,6 @@ import com.google.common.primitives.Ints; import com.google.inject.Inject; import com.google.protobuf.ByteString; -import com.hubspot.deploy.ExecutorData; import com.hubspot.deploy.ExecutorDataBuilder; import com.hubspot.mesos.JavaUtils; import com.hubspot.mesos.MesosUtils; @@ -48,7 +47,9 @@ import com.hubspot.mesos.SingularityDockerPortMapping; import com.hubspot.mesos.SingularityMesosTaskLabel; import com.hubspot.mesos.SingularityVolume; +import com.hubspot.singularity.SingularityS3UploaderFile; import com.hubspot.singularity.SingularityTask; +import com.hubspot.singularity.SingularityTaskExecutorData; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskRequest; import com.hubspot.singularity.config.SingularityConfiguration; @@ -357,12 +358,16 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit if (task.getDeploy().getExecutorData().isPresent()) { final ExecutorDataBuilder executorDataBldr = task.getDeploy().getExecutorData().get().toBuilder(); + String defaultS3Bucket = ""; + String s3UploaderKeyPattern = ""; if (configuration.getS3Configuration().isPresent()) { if (task.getRequest().getGroup().isPresent() && configuration.getS3Configuration().get().getGroupOverrides().containsKey(task.getRequest().getGroup().get())) { - final Optional loggingS3Bucket = Optional.of(configuration.getS3Configuration().get().getGroupOverrides().get(task.getRequest().getGroup().get()).getS3Bucket()); - LOG.trace("Setting loggingS3Bucket to {} for task {} executorData", loggingS3Bucket, taskId.getId()); - executorDataBldr.setLoggingS3Bucket(loggingS3Bucket); + defaultS3Bucket = configuration.getS3Configuration().get().getGroupOverrides().get(task.getRequest().getGroup().get()).getS3Bucket(); + LOG.trace("Setting defaultS3Bucket to {} for task {} executorData", defaultS3Bucket, taskId.getId()); + } else { + defaultS3Bucket = configuration.getS3Configuration().get().getS3Bucket(); } + s3UploaderKeyPattern = configuration.getS3Configuration().get().getS3KeyFormat(); } if (task.getPendingTask().getCmdLineArgsList().isPresent() && !task.getPendingTask().getCmdLineArgsList().get().isEmpty()) { @@ -376,7 +381,13 @@ private void prepareCustomExecutor(final TaskInfo.Builder bldr, final Singularit executorDataBldr.setExtraCmdLineArgs(extraCmdLineArgsBuilder.build()); } - final ExecutorData executorData = executorDataBldr.build(); + List uploaderAdditionalFiles = configuration.getS3Configuration().isPresent() ? configuration.getS3Configuration().get().getS3UploaderAdditionalFiles() : Collections.emptyList(); + Optional maybeS3StorageClass = configuration.getS3Configuration().isPresent() ? configuration.getS3Configuration().get().getS3StorageClass() : Optional.absent(); + Optional maybeApplyAfterBytes = configuration.getS3Configuration().isPresent() ? configuration.getS3Configuration().get().getApplyS3StorageClassAfterBytes() : Optional.absent(); + + final SingularityTaskExecutorData executorData = new SingularityTaskExecutorData(executorDataBldr.build(), uploaderAdditionalFiles, defaultS3Bucket, s3UploaderKeyPattern, + configuration.getCustomExecutorConfiguration().getServiceLog(), configuration.getCustomExecutorConfiguration().getServiceFinishedTailLog(), task.getRequest().getGroup(), + maybeS3StorageClass, maybeApplyAfterBytes); try { bldr.setData(ByteString.copyFromUtf8(objectMapper.writeValueAsString(executorData))); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java index 529ea2d0cd..eb90a79b20 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/S3LogResource.java @@ -1,34 +1,51 @@ package com.hubspot.singularity.resources; +import static com.hubspot.singularity.WebExceptions.checkBadRequest; import static com.hubspot.singularity.WebExceptions.checkNotFound; import static com.hubspot.singularity.WebExceptions.timeout; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import javax.ws.rs.Consumes; import javax.ws.rs.GET; +import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; -import org.jets3t.service.S3Service; -import org.jets3t.service.model.S3Object; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.HttpMethod; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.model.ResponseHeaderOverrides; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.base.Optional; +import com.google.common.base.Strings; import com.google.common.base.Throwables; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.primitives.Longs; @@ -47,12 +64,17 @@ import com.hubspot.singularity.SingularityRequestWithState; import com.hubspot.singularity.SingularityS3FormatHelper; import com.hubspot.singularity.SingularityS3Log; +import com.hubspot.singularity.SingularityS3LogMetadata; +import com.hubspot.singularity.SingularityS3UploaderFile; import com.hubspot.singularity.SingularityService; import com.hubspot.singularity.SingularityTaskHistory; import com.hubspot.singularity.SingularityTaskHistoryUpdate; import com.hubspot.singularity.SingularityTaskHistoryUpdate.SimplifiedTaskState; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityUser; +import com.hubspot.singularity.api.ContinuationToken; +import com.hubspot.singularity.api.SingularityS3SearchRequest; +import com.hubspot.singularity.api.SingularityS3SearchResult; import com.hubspot.singularity.auth.SingularityAuthorizationHelper; import com.hubspot.singularity.config.S3Configuration; import com.hubspot.singularity.data.DeployManager; @@ -60,6 +82,9 @@ import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.data.history.HistoryManager; import com.hubspot.singularity.data.history.RequestHistoryHelper; +import com.hubspot.singularity.helpers.S3ObjectSummaryHolder; +import com.hubspot.singularity.helpers.SingularityS3Service; +import com.hubspot.singularity.helpers.SingularityS3Services; import com.wordnik.swagger.annotations.Api; import com.wordnik.swagger.annotations.ApiOperation; import com.wordnik.swagger.annotations.ApiParam; @@ -70,36 +95,37 @@ public class S3LogResource extends AbstractHistoryResource { public static final String PATH = SingularityService.API_BASE_PATH + "/logs"; private static final Logger LOG = LoggerFactory.getLogger(S3LogResource.class); + private static final String CONTENT_DISPOSITION_DOWNLOAD_HEADER = "attachment"; + private static final String CONTENT_ENCODING_DOWNLOAD_HEADER = "identity"; + private static final String CONTINUATION_TOKEN_KEY_FORMAT = "%s-%s-%s"; + private static final int DEFAULT_TARGET_MAX_RESULTS = 10; - private static final String FORCE_DOWNLOAD_S3_PARAMS = "response-content-disposition=attachment&response-content-encoding=identity"; - - private final Optional s3ServiceDefault; - private final Map s3GroupOverride; + private final SingularityS3Services s3Services; private final Optional configuration; private final RequestHistoryHelper requestHistoryHelper; private final RequestManager requestManager; - private static final Comparator LOG_COMPARATOR = new Comparator() { + private static final Comparator LOG_COMPARATOR = new Comparator() { @Override - public int compare(SingularityS3Log o1, SingularityS3Log o2) { + public int compare(SingularityS3LogMetadata o1, SingularityS3LogMetadata o2) { return Longs.compare(o2.getLastModified(), o1.getLastModified()); } }; @Inject - public S3LogResource(RequestManager requestManager, HistoryManager historyManager, RequestHistoryHelper requestHistoryHelper, TaskManager taskManager, DeployManager deployManager, Optional s3ServiceDefault, - Optional configuration, SingularityAuthorizationHelper authorizationHelper, Optional user, Map s3GroupOverride) { + public S3LogResource(RequestManager requestManager, HistoryManager historyManager, RequestHistoryHelper requestHistoryHelper, TaskManager taskManager, DeployManager deployManager, + Optional configuration, SingularityAuthorizationHelper authorizationHelper, Optional user,SingularityS3Services s3Services) { super(historyManager, taskManager, deployManager, authorizationHelper, user); this.requestManager = requestManager; - this.s3ServiceDefault = s3ServiceDefault; this.configuration = configuration; this.requestHistoryHelper = requestHistoryHelper; - this.s3GroupOverride = s3GroupOverride; + this.s3Services = s3Services; } - private Collection getS3PrefixesForTask(S3Configuration s3Configuration, SingularityTaskId taskId, Optional startArg, Optional endArg) { + // Generation of prefixes + private Collection getS3PrefixesForTask(S3Configuration s3Configuration, SingularityTaskId taskId, Optional startArg, Optional endArg, String group) { Optional history = getTaskHistory(taskId); long start = taskId.getStartedAt(); @@ -128,7 +154,12 @@ private Collection getS3PrefixesForTask(S3Configuration s3Configuration, tag = history.get().getTask().getTaskRequest().getDeploy().getExecutorData().get().getLoggingTag(); } - Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), taskId, tag, start, end); + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), taskId, tag, start, end, group); + for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { + if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { + prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), taskId, tag, start, end, group)); + } + } LOG.trace("Task {} got S3 prefixes {} for start {}, end {}, tag {}", taskId, prefixes, start, end, tag); @@ -139,7 +170,7 @@ private boolean isCurrentDeploy(String requestId, String deployId) { return deployId.equals(deployManager.getInUseDeployId(requestId).orNull()); } - private Collection getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional startArg, Optional endArg) { + private Collection getS3PrefixesForRequest(S3Configuration s3Configuration, String requestId, Optional startArg, Optional endArg, String group) { Optional firstHistory = requestHistoryHelper.getFirstHistory(requestId); checkNotFound(firstHistory.isPresent(), "No request history found for %s", requestId); @@ -161,14 +192,19 @@ private Collection getS3PrefixesForRequest(S3Configuration s3Configurati end = Math.min(endArg.get(), end); } - Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end); + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, start, end, group); + for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { + if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { + prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, start, end, group)); + } + } LOG.trace("Request {} got S3 prefixes {} for start {}, end {}", requestId, prefixes, start, end); return prefixes; } - private Collection getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional startArg, Optional endArg) { + private Collection getS3PrefixesForDeploy(S3Configuration s3Configuration, String requestId, String deployId, Optional startArg, Optional endArg, String group) { SingularityDeployHistory deployHistory = getDeployHistory(requestId, deployId); long start = deployHistory.getDeployMarker().getTimestamp(); @@ -192,63 +228,214 @@ private Collection getS3PrefixesForDeploy(S3Configuration s3Configuratio tag = deployHistory.getDeploy().get().getExecutorData().get().getLoggingTag(); } - Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end); + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes(s3Configuration.getS3KeyFormat(), requestId, deployId, tag, start, end, group); + for (SingularityS3UploaderFile additionalFile : s3Configuration.getS3UploaderAdditionalFiles()) { + if (additionalFile.getS3UploaderKeyPattern().isPresent() && !additionalFile.getS3UploaderKeyPattern().get().equals(s3Configuration.getS3KeyFormat())) { + prefixes.addAll(SingularityS3FormatHelper.getS3KeyPrefixes(additionalFile.getS3UploaderKeyPattern().get(), requestId, deployId, tag, start, end, group)); + } + } LOG.trace("Request {}, deploy {} got S3 prefixes {} for start {}, end {}, tag {}", requestId, deployId, prefixes, start, end, tag); return prefixes; } - private List getS3LogsWithExecutorService(S3Configuration s3Configuration, Optional group, ListeningExecutorService executorService, Collection prefixes, final boolean excludeMetadata) throws InterruptedException, ExecutionException, TimeoutException { - List> futures = Lists.newArrayListWithCapacity(prefixes.size()); + private Map> getServiceToPrefixes(SingularityS3SearchRequest search) { + Map> servicesToPrefixes = new HashMap<>(); + + if (!search.getTaskIds().isEmpty()) { + for (String taskId : search.getTaskIds()) { + SingularityTaskId taskIdObject = getTaskIdObject(taskId); + String group = getRequestGroupForTask(taskIdObject).or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME); + Set s3Buckets = getBuckets(group); + Collection prefixes = getS3PrefixesForTask(configuration.get(), taskIdObject, search.getStart(), search.getEnd(), group); + for (String s3Bucket : s3Buckets) { + SingularityS3Service s3Service = s3Services.getServiceByGroupAndBucketOrDefault(group, s3Bucket); + if (!servicesToPrefixes.containsKey(s3Service)) { + servicesToPrefixes.put(s3Service, new HashSet()); + } + servicesToPrefixes.get(s3Service).addAll(prefixes); + } + } + } + if (!search.getRequestsAndDeploys().isEmpty()) { + for (Map.Entry> entry : search.getRequestsAndDeploys().entrySet()) { + String group = getRequestGroup(entry.getKey()).or(SingularityS3FormatHelper.DEFAULT_GROUP_NAME); + Set s3Buckets = getBuckets(group); + List prefixes = new ArrayList<>(); + if (!entry.getValue().isEmpty()) { + for (String deployId : entry.getValue()) { + prefixes.addAll(getS3PrefixesForDeploy(configuration.get(), entry.getKey(), deployId, search.getStart(), search.getEnd(), group)); + } + } else { + prefixes.addAll(getS3PrefixesForRequest(configuration.get(), entry.getKey(), search.getStart(), search.getEnd(), group)); + } + for (String s3Bucket : s3Buckets) { + SingularityS3Service s3Service = s3Services.getServiceByGroupAndBucketOrDefault(group, s3Bucket); + if (!servicesToPrefixes.containsKey(s3Service)) { + servicesToPrefixes.put(s3Service, new HashSet()); + } + servicesToPrefixes.get(s3Service).addAll(prefixes); + } + } + } - final String s3Bucket = (group.isPresent() && s3Configuration.getGroupOverrides().containsKey(group.get())) ? s3Configuration.getGroupOverrides().get(group.get()).getS3Bucket() : s3Configuration.getS3Bucket(); + // Trim prefixes to search. Less specific prefixes will contain all results of matching + more specific ones + for (Map.Entry> entry : servicesToPrefixes.entrySet()) { + Set results = new HashSet<>(); + boolean contains = false; + for (String prefix : entry.getValue()) { + for (String unique : results) { + if (prefix.startsWith(unique) && prefix.length() > unique.length()) { + contains = true; + break; + } else if (unique.startsWith(prefix) && unique.length() > prefix.length()) { + results.remove(unique); + results.add(prefix); + contains = true; + break; + } + } + if (!contains) { + results.add(prefix); + } + } + entry.getValue().retainAll(results); + } - final S3Service s3Service = (group.isPresent() && s3GroupOverride.containsKey(group.get())) ? s3GroupOverride.get(group.get()) : s3ServiceDefault.get(); + return servicesToPrefixes; + } - for (final String s3Prefix : prefixes) { - futures.add(executorService.submit(new Callable() { + private Set getBuckets(String group) { + Set s3Buckets = new HashSet<>(); + s3Buckets.add(configuration.get().getGroupOverrides().containsKey(group) ? configuration.get().getGroupOverrides().get(group).getS3Bucket() : configuration.get().getS3Bucket()); + s3Buckets.add(configuration.get().getGroupS3SearchConfigs().containsKey(group) ? configuration.get().getGroupS3SearchConfigs().get(group).getS3Bucket() : configuration.get().getS3Bucket()); + for (SingularityS3UploaderFile uploaderFile : configuration.get().getS3UploaderAdditionalFiles()) { + if (uploaderFile.getS3UploaderBucket().isPresent() && !s3Buckets.contains(uploaderFile.getS3UploaderBucket().get())) { + s3Buckets.add(uploaderFile.getS3UploaderBucket().get()); + } + } + return s3Buckets; + } - @Override - public S3Object[] call() throws Exception { - return s3Service.listObjects(s3Bucket, s3Prefix, null); + // Fetching logs + private List getS3LogsWithExecutorService(S3Configuration s3Configuration, ListeningExecutorService executorService, Map> servicesToPrefixes, int totalPrefixCount, final SingularityS3SearchRequest search, final ConcurrentHashMap continuationTokens, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException { + List>> futures = Lists.newArrayListWithCapacity(totalPrefixCount); + + final AtomicInteger resultCount = new AtomicInteger(); + + for (final Map.Entry> entry : servicesToPrefixes.entrySet()) { + final String s3Bucket = entry.getKey().getBucket(); + final String group = entry.getKey().getGroup(); + final AmazonS3 s3Client = entry.getKey().getS3Client(); + + for (final String s3Prefix : entry.getValue()) { + final String key = String.format(CONTINUATION_TOKEN_KEY_FORMAT, group, s3Bucket, s3Prefix); + if (search.getContinuationTokens().containsKey(key) && search.getContinuationTokens().get(key).isLastPage()) { + LOG.trace("No further content for prefix {} in bucket {}, skipping", s3Prefix, s3Bucket); + continuationTokens.putIfAbsent(key, search.getContinuationTokens().get(key)); + continue; } - })); + futures.add(executorService.submit(new Callable>() { + + @Override + public List call() throws Exception { + ListObjectsV2Request request = new ListObjectsV2Request().withBucketName(s3Bucket).withPrefix(s3Prefix); + if (paginated) { + Optional token = Optional.absent(); + if (search.getContinuationTokens().containsKey(key) && !Strings.isNullOrEmpty(search.getContinuationTokens().get(key).getValue())) { + request.setContinuationToken(search.getContinuationTokens().get(key).getValue()); + token = Optional.of(search.getContinuationTokens().get(key)); + } + int targetResultCount = search.getMaxPerPage().or(DEFAULT_TARGET_MAX_RESULTS); + request.setMaxKeys(targetResultCount); + if (resultCount.get() < targetResultCount) { + ListObjectsV2Result result = s3Client.listObjectsV2(request); + if (result.getObjectSummaries().isEmpty()) { + continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), true)); + return Collections.emptyList(); + } else { + boolean addToList = incrementIfLessThan(resultCount, result.getObjectSummaries().size(), targetResultCount); + if (addToList) { + continuationTokens.putIfAbsent(key, new ContinuationToken(result.getNextContinuationToken(), !result.isTruncated())); + List objectSummaryHolders = new ArrayList<>(); + for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { + objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary)); + } + return objectSummaryHolders; + } else { + continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false))); + return Collections.emptyList(); + } + } + } else { + continuationTokens.putIfAbsent(key, token.or(new ContinuationToken(null, false))); + return Collections.emptyList(); + } + } else { + ListObjectsV2Result result = s3Client.listObjectsV2(request); + List objectSummaryHolders = new ArrayList<>(); + for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { + objectSummaryHolders.add(new S3ObjectSummaryHolder(group, objectSummary)); + } + return objectSummaryHolders; + } + } + })); + } } final long start = System.currentTimeMillis(); - List results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS); + List> results = Futures.allAsList(futures).get(s3Configuration.getWaitForS3ListSeconds(), TimeUnit.SECONDS); - List objects = Lists.newArrayListWithExpectedSize(results.size() * 2); + List objects = Lists.newArrayListWithExpectedSize(results.size() * 2); - for (S3Object[] s3Objects : results) { - for (final S3Object s3Object : s3Objects) { - objects.add(s3Object); + for (List s3ObjectSummaryHolders : results) { + for (final S3ObjectSummaryHolder s3ObjectHolder : s3ObjectSummaryHolders) { + objects.add(s3ObjectHolder); } } LOG.trace("Got {} objects from S3 after {}", objects.size(), JavaUtils.duration(start)); - List> logFutures = Lists.newArrayListWithCapacity(objects.size()); + List> logFutures = Lists.newArrayListWithCapacity(objects.size()); final Date expireAt = new Date(System.currentTimeMillis() + s3Configuration.getExpireS3LinksAfterMillis()); - for (final S3Object s3Object : objects) { - logFutures.add(executorService.submit(new Callable() { + for (final S3ObjectSummaryHolder s3ObjectHolder : objects) { + final S3ObjectSummary s3Object = s3ObjectHolder.getObjectSummary(); + final AmazonS3 s3Client = s3Services.getServiceByGroupAndBucketOrDefault(s3ObjectHolder.getGroup(), s3Object.getBucketName()).getS3Client(); + logFutures.add(executorService.submit(new Callable() { @Override - public SingularityS3Log call() throws Exception { - String getUrl = s3Service.createSignedGetUrl(s3Bucket, s3Object.getKey(), expireAt); - String downloadUrl = s3Service.createSignedUrl("GET", s3Bucket, s3Object.getKey(), FORCE_DOWNLOAD_S3_PARAMS, null, expireAt.getTime() / 1000, false); - + public SingularityS3LogMetadata call() throws Exception { Optional maybeStartTime = Optional.absent(); Optional maybeEndTime = Optional.absent(); - if (!excludeMetadata) { - Map objectMetadata = s3Service.getObjectDetails(s3Bucket, s3Object.getKey()).getMetadataMap(); + if (!search.isExcludeMetadata()) { + GetObjectMetadataRequest metadataRequest = new GetObjectMetadataRequest(s3Object.getBucketName(), s3Object.getKey()); + Map objectMetadata = s3Client.getObjectMetadata(metadataRequest).getUserMetadata(); maybeStartTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_START_S3_ATTR); maybeEndTime = getMetadataAsLong(objectMetadata, SingularityS3Log.LOG_END_S3_ATTR); } - return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModifiedDate().getTime(), s3Object.getContentLength(), downloadUrl, maybeStartTime, maybeEndTime); + if (search.isListOnly()) { + return new SingularityS3LogMetadata(s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), maybeStartTime, maybeEndTime); + } else { + GeneratePresignedUrlRequest getUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) + .withMethod(HttpMethod.GET) + .withExpiration(expireAt); + String getUrl = s3Client.generatePresignedUrl(getUrlRequest).toString(); + + ResponseHeaderOverrides downloadHeaders = new ResponseHeaderOverrides(); + downloadHeaders.setContentDisposition(CONTENT_DISPOSITION_DOWNLOAD_HEADER); + downloadHeaders.setContentEncoding(CONTENT_ENCODING_DOWNLOAD_HEADER); + GeneratePresignedUrlRequest downloadUrlRequest = new GeneratePresignedUrlRequest(s3Object.getBucketName(), s3Object.getKey()) + .withMethod(HttpMethod.GET) + .withExpiration(expireAt) + .withResponseHeaders(downloadHeaders); + String downloadUrl = s3Client.generatePresignedUrl(downloadUrlRequest).toString(); + + return new SingularityS3Log(getUrl, s3Object.getKey(), s3Object.getLastModified().getTime(), s3Object.getSize(), downloadUrl, maybeStartTime, maybeEndTime); + } } })); @@ -257,15 +444,23 @@ public SingularityS3Log call() throws Exception { return Futures.allAsList(logFutures).get(s3Configuration.getWaitForS3LinksSeconds(), TimeUnit.SECONDS); } - private Optional getMetadataAsLong(Map objectMetadata, String keyName) { + private boolean incrementIfLessThan(AtomicInteger count, int add, int threshold) { + while (true) { + int current = count.get(); + if (current >= threshold) { + return false; + } + if (count.compareAndSet(current, current + add)) { + return true; + } + } + } + + private Optional getMetadataAsLong(Map objectMetadata, String keyName) { try { if (objectMetadata.containsKey(keyName)) { Object maybeLong = objectMetadata.get(keyName); - if (maybeLong instanceof String) { - return Optional.of(Long.parseLong((String) maybeLong)); - } else { - return Optional.of((Long) maybeLong); - } + return Optional.of(Long.parseLong((String) maybeLong)); } else { return Optional.absent(); } @@ -274,28 +469,39 @@ private Optional getMetadataAsLong(Map objectMetadata, Str } } - private List getS3Logs(S3Configuration s3Configuration, Optional group, Collection prefixes, boolean excldueMetadata) throws InterruptedException, ExecutionException, TimeoutException { - if (prefixes.isEmpty()) { - return Collections.emptyList(); + private SingularityS3SearchResult getS3Logs(S3Configuration s3Configuration, Map> servicesToPrefixes, final SingularityS3SearchRequest search, final boolean paginated) throws InterruptedException, ExecutionException, TimeoutException { + int totalPrefixCount = 0; + for (Map.Entry> entry : servicesToPrefixes.entrySet()) { + totalPrefixCount += entry.getValue().size(); } - ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.min(prefixes.size(), s3Configuration.getMaxS3Threads()), + if (totalPrefixCount == 0) { + return SingularityS3SearchResult.empty(); + } + + ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(Math.min(totalPrefixCount, s3Configuration.getMaxS3Threads()), new ThreadFactoryBuilder().setNameFormat("S3LogFetcher-%d").build())); try { - List logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, group, executorService, prefixes, excldueMetadata)); + final ConcurrentHashMap continuationTokens = new ConcurrentHashMap<>(); + List logs = Lists.newArrayList(getS3LogsWithExecutorService(s3Configuration, executorService, servicesToPrefixes, totalPrefixCount, search, continuationTokens, paginated)); Collections.sort(logs, LOG_COMPARATOR); - return logs; + return new SingularityS3SearchResult(continuationTokens, isFinalPageForAllPrefixes(continuationTokens.values()), logs); } finally { executorService.shutdownNow(); } } - private void checkS3() { - checkNotFound(s3ServiceDefault.isPresent(), "S3 configuration was absent"); - checkNotFound(configuration.isPresent(), "S3 configuration was absent"); + private boolean isFinalPageForAllPrefixes(Collection continuationTokens) { + for (ContinuationToken token : continuationTokens) { + if (!token.isLastPage()) { + return false; + } + } + return true; } + // Finding request group private Optional getRequestGroupForTask(final SingularityTaskId taskId) { Optional maybeTaskHistory = getTaskHistory(taskId); if (maybeTaskHistory.isPresent()) { @@ -325,20 +531,34 @@ private Optional getRequestGroup(final String requestId) { } } + private void checkS3() { + checkNotFound(s3Services.isS3ConfigPresent(), "S3 configuration was absent"); + checkNotFound(configuration.isPresent(), "S3 configuration was absent"); + } + @GET @Path("/task/{taskId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific task.") - public List getS3LogsForTask( + public List getS3LogsForTask( @ApiParam("The task ID to search for") @PathParam("taskId") String taskId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, - @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") boolean excludeMetadata, + @ApiParam("Do not generate download/get urls, only list the files and metadata") @QueryParam("list") boolean listOnly) throws Exception { checkS3(); - SingularityTaskId taskIdObject = getTaskIdObject(taskId); + final SingularityS3SearchRequest search = new SingularityS3SearchRequest( + Collections.>emptyMap(), + Collections.singletonList(taskId), + start, + end, + excludeMetadata, + listOnly, + Optional.absent(), + Collections.emptyMap()); try { - return getS3Logs(configuration.get(), getRequestGroupForTask(taskIdObject), getS3PrefixesForTask(configuration.get(), taskIdObject, start, end), excludeMetadata.or(false)); + return getS3Logs(configuration.get(), getServiceToPrefixes(search), search, false).getResults(); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", taskId); } catch (Throwable t) { @@ -349,15 +569,27 @@ public List getS3LogsForTask( @GET @Path("/request/{requestId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific request.") - public List getS3LogsForRequest( + public List getS3LogsForRequest( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, - @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") boolean excludeMetadata, + @ApiParam("Do not generate download/get urls, only list the files and metadata") @QueryParam("list") boolean listOnly, + @ApiParam("Max number of results to return per bucket searched") @QueryParam("maxPerPage") Optional maxPerPage) throws Exception { checkS3(); try { - return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForRequest(configuration.get(), requestId, start, end), excludeMetadata.or(false)); + final SingularityS3SearchRequest search = new SingularityS3SearchRequest( + ImmutableMap.of(requestId, Collections.emptyList()), + Collections.emptyList(), + start, + end, + excludeMetadata, + listOnly, + Optional.absent(), + Collections.emptyMap()); + + return getS3Logs(configuration.get(), getServiceToPrefixes(search), search, false).getResults(); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s", requestId); } catch (Throwable t) { @@ -368,16 +600,28 @@ public List getS3LogsForRequest( @GET @Path("/request/{requestId}/deploy/{deployId}") @ApiOperation("Retrieve the list of logs stored in S3 for a specific deploy.") - public List getS3LogsForDeploy( + public List getS3LogsForDeploy( @ApiParam("The request ID to search for") @PathParam("requestId") String requestId, @ApiParam("The deploy ID to search for") @PathParam("deployId") String deployId, @ApiParam("Start timestamp (millis, 13 digit)") @QueryParam("start") Optional start, @ApiParam("End timestamp (mills, 13 digit)") @QueryParam("end") Optional end, - @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") Optional excludeMetadata) throws Exception { + @ApiParam("Exclude custom object metadata") @QueryParam("excludeMetadata") boolean excludeMetadata, + @ApiParam("Do not generate download/get urls, only list the files and metadata") @QueryParam("list") boolean listOnly, + @ApiParam("Max number of results to return per bucket searched") @QueryParam("maxPerPage") Optional maxPerPage) throws Exception { checkS3(); try { - return getS3Logs(configuration.get(), getRequestGroup(requestId), getS3PrefixesForDeploy(configuration.get(), requestId, deployId, start, end), excludeMetadata.or(false)); + final SingularityS3SearchRequest search = new SingularityS3SearchRequest( + ImmutableMap.of(requestId, Collections.singletonList(deployId)), + Collections.emptyList(), + start, + end, + excludeMetadata, + listOnly, + Optional.absent(), + Collections.emptyMap()); + + return getS3Logs(configuration.get(), getServiceToPrefixes(search), search, false).getResults(); } catch (TimeoutException te) { throw timeout("Timed out waiting for response from S3 for %s-%s", requestId, deployId); } catch (Throwable t) { @@ -385,4 +629,21 @@ public List getS3LogsForDeploy( } } + @POST + @Path("/search") + @Consumes(MediaType.APPLICATION_JSON) + @ApiOperation("Retrieve a paginated list of logs stored in S3") + public SingularityS3SearchResult getPaginatedS3Logs(@ApiParam(required = true) SingularityS3SearchRequest search) throws Exception { + checkS3(); + + checkBadRequest(!search.getRequestsAndDeploys().isEmpty() || !search.getTaskIds().isEmpty(), "Must specify at least one request or task to search"); + + try { + return getS3Logs(configuration.get(), getServiceToPrefixes(search), search, true); + } catch (TimeoutException te) { + throw timeout("Timed out waiting for response from S3 for %s", search); + } catch (Throwable t) { + throw Throwables.propagate(t); + } + } } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java index 407c70984b..0377d40660 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularityS3Test.java @@ -16,27 +16,31 @@ public void testS3FormatHelper() throws Exception { SingularityTaskId taskId = new SingularityTaskId("rid", "did", 1, 1, "host", "rack"); - long start = 1414610537117l; // Wed, 29 Oct 2014 19:22:17 GMT - long end = 1415724215000l; // Tue, 11 Nov 2014 16:43:35 GMT + long start = 1414610537117L; // Wed, 29 Oct 2014 19:22:17 GMT + long end = 1415724215000L; // Tue, 11 Nov 2014 16:43:35 GMT - Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%taskId", taskId, Optional. absent(), start, end); + Collection prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%taskId", taskId, Optional. absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); - end = 1447265861000l; // Tue, 11 Nov 2015 16:43:35 GMT + end = 1447265861000L; // Tue, 11 Nov 2015 16:43:35 GMT - prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%taskId", taskId, Optional. absent(), start, end); + prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%taskId", taskId, Optional. absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); - start = 1415750399999l; - end = 1415771999000l; + start = 1415750399999L; + end = 1415771999000L; - prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%d/%taskId", taskId, Optional. absent(), start, end); + prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%Y/%m/%d/%taskId", taskId, Optional. absent(), start, end, "default"); Assert.assertTrue(prefixes.size() == 2); - final long NOV2014TUES11 = 1415724215000l; + prefixes = SingularityS3FormatHelper.getS3KeyPrefixes("%requestId/%group/%Y/%m", taskId, Optional. absent(), start, end, "groupName"); + + Assert.assertEquals("rid/groupName/2014/11", prefixes.iterator().next()); + + final long NOV2014TUES11 = 1415724215000L; Assert.assertEquals("wat-hostname", SingularityS3FormatHelper.getKey("wat-%host", 0, System.currentTimeMillis(), "filename", "hostname")); Assert.assertEquals("file1.txt-2", SingularityS3FormatHelper.getKey("%filename-%index", 2, System.currentTimeMillis(), "file1.txt", "hostname")); diff --git a/book.json b/book.json index 8755df9e25..fc11045703 100644 --- a/book.json +++ b/book.json @@ -10,7 +10,7 @@ "contribute": "https://github.com/HubSpot/Singularity" }, "variables": { - "releases": ["0.13.0", "0.12.0", "0.11.0", "0.10.1", "0.10.0", "0.9.0", "0.8.0", "0.7.1", "0.7.0", "0.6.2", "0.6.1", "0.6.0", "0.5.0", "0.4.11", "0.4.10", "0.4.9", "0.4.8", "0.4.7", "0.4.6", "0.4.5"] + "releases": ["0.14.0", "0.13.0", "0.12.0", "0.11.0", "0.10.1", "0.10.0", "0.9.0", "0.8.0", "0.7.1", "0.7.0", "0.6.2", "0.6.1", "0.6.0", "0.5.0", "0.4.11", "0.4.10", "0.4.9", "0.4.8", "0.4.7", "0.4.6", "0.4.5"] }, "plugins": ["github", "-sharing", "-livereload", "-highlight", "copy-code-button"], "pluginsConfig": { diff --git a/pom.xml b/pom.xml index ef1d9c87c8..318c135b59 100644 --- a/pom.xml +++ b/pom.xml @@ -43,6 +43,7 @@ 0.28.1-2.0.20.ubuntu1404 0.28.2 2 + 1.10.77 @@ -244,6 +245,22 @@ rfc5545-datetime 0.2.2 + + + com.amazonaws + aws-java-sdk-s3 + ${aws.sdk.version} + + + commons-logging + commons-logging + + + joda-time + joda-time + + +