-
Notifications
You must be signed in to change notification settings - Fork 188
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1391 from HubSpot/s3_rework
S3 Search Improvements
- Loading branch information
Showing
41 changed files
with
1,569 additions
and
591 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| ## Changes in `0.14.0` | ||
|
|
||
| Check out the [0.14.0 milestone](https://github.com/HubSpot/Singularity/issues?q=milestone%3A0.14.0+is%3Aclosed) to see new features / bugfixes in detail. | ||
|
|
||
| ## Configuration Changes | ||
|
|
||
| [#1391](https://github.com/HubSpot/Singularity/pull/1391) include a rework of some of the S3 settings in Singularity. If you use the `SingularityExecutor` or `SingularityExecutorCleanup` modules and use the S3 upload features, you will need an update to your configuration. The fields for specifying which files to upload have been moved out of the `SingularityExecutor`. An example below shows all fields that would move. | ||
|
|
||
| Old Configuration (gets removed from `SingularityExecutor` and `SingularityExecutorCleanup` yaml files) | ||
| ```yaml | ||
| executor: | ||
| s3UploaderBucket: my-logs-bucket | ||
| s3UploaderKeyPattern: "%requestId/%Y/%m/%taskId_%index-%s-%filename" | ||
| s3UploaderAdditionalFiles: | ||
| - access.log | ||
| s3StorageClass: "STANDARD_IA" | ||
| applyS3StorageClassAfterBytes: 75000 | ||
| ``` | ||
| New Configuration (if not already present for use with S3 log searching) | ||
| ```yaml | ||
| # in SingularityExecutorCleanup yaml configuration | ||
| executorCleanup: | ||
| defaultS3Bucket: my-logs-bucket | ||
| s3KeyFormat: "%requestId/%Y/%m/%taskId_%index-%s-%filename" | ||
| s3StorageClass: "STANDARD_IA" | ||
| applyS3StorageClassAfterBytes: 75000 | ||
| s3UploaderAdditionalFiles: | ||
| - filename: access.log | ||
| # The default directory in the executor was set to 'logs', now it must be manually specified | ||
| # If not specified, the directory to search for log files will be the task app directory in the sandbox | ||
| directory: logs | ||
|
|
||
| # in SingularityService yaml configuration | ||
| s3: | ||
| s3Bucket: my-logs-bucket | ||
| s3KeyFormat: "%requestId/%Y/%m/%taskId_%index-%s-%filename" | ||
| s3StorageClass: "STANDARD_IA" | ||
| applyS3StorageClassAfterBytes: 75000 | ||
| s3UploaderAdditionalFiles: | ||
| - filename: access.log | ||
| # The default directory in the executor was set to 'logs', now it must be manually specified | ||
| # If not specified, the directory to search for log files will be the task app directory in the sandbox | ||
| directory: logs | ||
| ``` | ||
| **NOTE** - To upgrade smoothly, it is strongly recommended to deploy `SingularityService` and the `SingularityExecutorCleanup` *before* deploying the `SingularityExecutor` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
63 changes: 63 additions & 0 deletions
63
SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3LogMetadata.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| package com.hubspot.singularity; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonCreator; | ||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import com.google.common.base.Optional; | ||
| import com.wordnik.swagger.annotations.ApiModelProperty; | ||
|
|
||
| public class SingularityS3LogMetadata { | ||
| public static final String LOG_START_S3_ATTR = "starttime"; | ||
| public static final String LOG_END_S3_ATTR = "endtime"; | ||
|
|
||
| private final String key; | ||
| private final long lastModified; | ||
| private final long size; | ||
| private final Optional<Long> startTime; | ||
| private final Optional<Long> endTime; | ||
|
|
||
| @JsonCreator | ||
| public SingularityS3LogMetadata(@JsonProperty("key") String key, @JsonProperty("lastModified") long lastModified, @JsonProperty("size") long size, | ||
| @JsonProperty("startTime") Optional<Long> startTime, @JsonProperty("endTime") Optional<Long> endTime) { | ||
| this.key = key; | ||
| this.lastModified = lastModified; | ||
| this.size = size; | ||
| this.startTime = startTime; | ||
| this.endTime = endTime; | ||
| } | ||
|
|
||
| @ApiModelProperty("S3 key") | ||
| public String getKey() { | ||
| return key; | ||
| } | ||
|
|
||
| @ApiModelProperty("Last modified time") | ||
| public long getLastModified() { | ||
| return lastModified; | ||
| } | ||
|
|
||
| @ApiModelProperty("File size (in bytes)") | ||
| public long getSize() { | ||
| return size; | ||
| } | ||
|
|
||
| @ApiModelProperty("Time the log file started being written to") | ||
| public Optional<Long> getStartTime() { | ||
| return startTime; | ||
| } | ||
|
|
||
| @ApiModelProperty("Time the log file was finished being written to") | ||
| public Optional<Long> getEndTime() { | ||
| return endTime; | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return "SingularityS3Log{" + | ||
| "key='" + key + '\'' + | ||
| ", lastModified=" + lastModified + | ||
| ", size=" + size + | ||
| ", startTime=" + startTime + | ||
| ", endTime=" + endTime + | ||
| '}'; | ||
| } | ||
| } |
Oops, something went wrong.