diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/executor/SingularityExecutorLogrotateFrequency.java b/SingularityBase/src/main/java/com/hubspot/singularity/executor/SingularityExecutorLogrotateFrequency.java index 6562a0f1f2..5740667d93 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/executor/SingularityExecutorLogrotateFrequency.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/executor/SingularityExecutorLogrotateFrequency.java @@ -4,9 +4,9 @@ public enum SingularityExecutorLogrotateFrequency { HOURLY("daily", Optional.of("0 * * * *")), // we have to use the "daily" frequency because not all versions of logrotate support "hourly" - DAILY("daily", Optional.absent()), - WEEKLY("weekly", Optional.absent()), - MONTHLY("monthly", Optional.absent()); + DAILY("daily", Optional.absent()), + WEEKLY("weekly", Optional.absent()), + MONTHLY("monthly", Optional.absent()); private final String logrotateValue; private final Optional cronSchedule; diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/TemplateManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/TemplateManager.java index 7e4ec6bab1..88ff20f623 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/TemplateManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/TemplateManager.java @@ -25,6 +25,7 @@ public class TemplateManager { private final Template runnerTemplate; private final Template environmentTemplate; private final Template logrotateTemplate; + private final Template logrotateHourlyTemplate; private final Template logrotateCronTemplate; private final Template dockerTemplate; @@ -32,12 +33,14 @@ public class TemplateManager { public TemplateManager(@Named(SingularityExecutorModule.RUNNER_TEMPLATE) Template runnerTemplate, @Named(SingularityExecutorModule.ENVIRONMENT_TEMPLATE) Template environmentTemplate, @Named(SingularityExecutorModule.LOGROTATE_TEMPLATE) Template logrotateTemplate, + @Named(SingularityExecutorModule.LOGROTATE_HOURLY_TEMPLATE) Template logrotateHourlyTemplate, @Named(SingularityExecutorModule.LOGROTATE_CRON_TEMPLATE) Template logrotateCronTemplate, @Named(SingularityExecutorModule.DOCKER_TEMPLATE) Template dockerTemplate ) { this.runnerTemplate = runnerTemplate; this.environmentTemplate = environmentTemplate; this.logrotateTemplate = logrotateTemplate; + this.logrotateHourlyTemplate = logrotateHourlyTemplate; this.logrotateCronTemplate = logrotateCronTemplate; this.dockerTemplate = dockerTemplate; } @@ -54,6 +57,10 @@ public void writeLogrotateFile(Path destination, LogrotateTemplateContext logRot writeTemplate(destination, logrotateTemplate, logRotateContext); } + public void writeHourlyLogrotateFile(Path destination, LogrotateTemplateContext logRotateContext) { + writeTemplate(destination, logrotateHourlyTemplate, logRotateContext); + } + public boolean writeCronEntryForLogrotate(Path destination, LogrotateCronTemplateContext logrotateCronTemplateContext) { writeTemplate(destination, logrotateCronTemplate, logrotateCronTemplateContext); final File destinationFile = destination.toFile(); diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java index f99de56d2c..d1d0fa40dd 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorConfiguration.java @@ -99,6 +99,10 @@ public class SingularityExecutorConfiguration extends BaseRunnerConfiguration { @JsonProperty private String logrotateConfDirectory = "/etc/logrotate.d"; + @NotEmpty + @JsonProperty + private String logrotateHourlyConfDirectory = "/etc/logrotate.d/hourly"; + @NotEmpty @JsonProperty private String logrotateToDirectory = "logs"; @@ -328,6 +332,10 @@ public String getLogrotateConfDirectory() { return logrotateConfDirectory; } + public String getLogrotateHourlyConfDirectory() { + return logrotateHourlyConfDirectory; + } + public String getLogrotateToDirectory() { return logrotateToDirectory; } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorLogrotateAdditionalFile.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorLogrotateAdditionalFile.java index 633636d399..9ba201b27a 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorLogrotateAdditionalFile.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorLogrotateAdditionalFile.java @@ -3,24 +3,28 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Optional; +import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; public class SingularityExecutorLogrotateAdditionalFile { private final String filename; private final Optional extension; private final Optional dateformat; + private final Optional logrotateFrequencyOverride; @JsonCreator public static SingularityExecutorLogrotateAdditionalFile fromString(String value) { - return new SingularityExecutorLogrotateAdditionalFile(value, Optional.absent(), Optional.absent()); + return new SingularityExecutorLogrotateAdditionalFile(value, Optional.absent(), Optional.absent(), null); } @JsonCreator public SingularityExecutorLogrotateAdditionalFile(@JsonProperty("filename") String filename, - @JsonProperty("extension") Optional extension, - @JsonProperty("dateformat") Optional dateformat) { + @JsonProperty("extension") Optional extension, + @JsonProperty("dateformat") Optional dateformat, + @JsonProperty("logrotateFrequencyOverride") SingularityExecutorLogrotateFrequency logrotateFrequencyOverride) { this.filename = filename; this.extension = extension; this.dateformat = dateformat; + this.logrotateFrequencyOverride = Optional.fromNullable(logrotateFrequencyOverride); } public String getFilename() { @@ -34,4 +38,9 @@ public Optional getExtension() { public Optional getDateformat() { return dateformat; } + + public Optional getLogrotateFrequencyOverride() { + return logrotateFrequencyOverride; + } + } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java index ee7e1ee00b..31327b4e2e 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/config/SingularityExecutorModule.java @@ -30,6 +30,7 @@ public class SingularityExecutorModule extends AbstractModule { public static final String RUNNER_TEMPLATE = "runner.sh"; public static final String ENVIRONMENT_TEMPLATE = "deploy.env"; public static final String LOGROTATE_TEMPLATE = "logrotate.conf"; + public static final String LOGROTATE_HOURLY_TEMPLATE = "logrotate.hourly.conf"; public static final String LOGROTATE_CRON_TEMPLATE = "logrotate.cron"; public static final String DOCKER_TEMPLATE = "docker.sh"; public static final String LOCAL_DOWNLOAD_HTTP_CLIENT = "SingularityExecutorModule.local.download.http.client"; @@ -72,6 +73,13 @@ public Template providesLogrotateTemplate(Handlebars handlebars) throws IOExcept return handlebars.compile(LOGROTATE_TEMPLATE); } + @Provides + @Singleton + @Named(LOGROTATE_HOURLY_TEMPLATE) + public Template providesLogrotateHourlyTemplate(Handlebars handlebars) throws IOException { + return handlebars.compile(LOGROTATE_HOURLY_TEMPLATE); + } + @Provides @Singleton @Named(LOGROTATE_CRON_TEMPLATE) diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateAdditionalFile.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateAdditionalFile.java index 3b59ea1818..a3a9326440 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateAdditionalFile.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateAdditionalFile.java @@ -1,14 +1,19 @@ package com.hubspot.singularity.executor.models; +import com.google.common.base.Optional; +import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; + public class LogrotateAdditionalFile { private final String filename; private final String extension; private final String dateformat; + private final Optional logrotateFrequencyOverride; - public LogrotateAdditionalFile(String filename, String extension, String dateformat) { + public LogrotateAdditionalFile(String filename, String extension, String dateformat, Optional logrotateFrequencyOverride) { this.filename = filename; this.extension = extension; this.dateformat = dateformat; + this.logrotateFrequencyOverride = logrotateFrequencyOverride; } public String getFilename() { @@ -23,12 +28,18 @@ public String getDateformat() { return dateformat; } + public String getLogrotateFrequencyOverride() { + return logrotateFrequencyOverride.isPresent() ? + logrotateFrequencyOverride.get().getLogrotateValue() : ""; + } + @Override public String toString() { return "LogrotateAdditionalFile{" + "filename='" + filename + '\'' + ", extension='" + extension + '\'' + ", dateformat='" + dateformat + '\'' + + ", frequency='" + logrotateFrequencyOverride + '\'' + '}'; } } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateCronTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateCronTemplateContext.java index 19f9dfe5e6..33d0df6b8c 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateCronTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateCronTemplateContext.java @@ -15,7 +15,7 @@ public class LogrotateCronTemplateContext { public LogrotateCronTemplateContext(SingularityExecutorConfiguration configuration, SingularityExecutorTaskDefinition taskDefinition, SingularityExecutorLogrotateFrequency logrotateFrequency) { this.logrotateCommand = configuration.getLogrotateCommand(); this.logrotateStateFile = taskDefinition.getLogrotateStateFilePath().toString(); - this.logrotateConfig = Paths.get(configuration.getLogrotateConfDirectory()).resolve(taskDefinition.getTaskId()).toString(); + this.logrotateConfig = Paths.get(configuration.getLogrotateHourlyConfDirectory()).resolve(taskDefinition.getTaskId()).toString(); this.cronSchedule = logrotateFrequency.getCronSchedule().get(); } diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java index 49aa7b2ca7..acf5760b75 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java @@ -2,10 +2,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import com.google.common.base.Optional; import com.google.common.base.Strings; import com.google.common.io.Files; +import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; import com.hubspot.singularity.executor.config.SingularityExecutorLogrotateAdditionalFile; import com.hubspot.singularity.executor.task.SingularityExecutorTaskDefinition; @@ -69,10 +71,30 @@ public String getCompressExt() { } /** - * Extra files for logrotate to rotate. If these do not exist logrotate will continue without error. + * Extra files for logrotate to rotate (non-hourly). If these do not exist logrotate will continue without error. * @return filenames to rotate. */ public List getExtrasFiles() { + return getAllExtraFiles().stream() + .filter(p -> !p.getLogrotateFrequencyOverride().equals(SingularityExecutorLogrotateFrequency.HOURLY.getLogrotateValue())) + .collect(Collectors.toList()); + } + + /** + * Extra files for logrotate to rotate hourly. If these do not exist logrotate will continue without error. + * @return filenames to rotate. + */ + public List getExtrasFilesHourly() { + return getAllExtraFiles().stream() + .filter(p -> p.getLogrotateFrequencyOverride().equals(SingularityExecutorLogrotateFrequency.HOURLY.getLogrotateValue())) + .collect(Collectors.toList()); + } + + public boolean isGlobalLogrotateHourly() { + return configuration.getLogrotateFrequency().getLogrotateValue().equals(SingularityExecutorLogrotateFrequency.HOURLY.getLogrotateValue()); + } + + private List getAllExtraFiles() { final List original = configuration.getLogrotateAdditionalFiles(); final List transformed = new ArrayList<>(original.size()); @@ -85,11 +107,12 @@ public List getExtrasFiles() { } transformed.add( - new LogrotateAdditionalFile( - taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getFilename()).toString(), - additionalFile.getExtension().or(Strings.emptyToNull(Files.getFileExtension(additionalFile.getFilename()))), - dateformat - )); + new LogrotateAdditionalFile( + taskDefinition.getTaskDirectoryPath().resolve(additionalFile.getFilename()).toString(), + additionalFile.getExtension().or(Strings.emptyToNull(Files.getFileExtension(additionalFile.getFilename()))), + dateformat, + additionalFile.getLogrotateFrequencyOverride() + )); } return transformed; diff --git a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java index ad63f1a2c8..486dd21ca1 100644 --- a/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java +++ b/SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java @@ -1,5 +1,6 @@ package com.hubspot.singularity.executor.task; +import java.io.File; import java.lang.ProcessBuilder.Redirect; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -15,11 +16,12 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.hubspot.singularity.SingularityS3FormatHelper; +import com.hubspot.singularity.SingularityS3UploaderFile; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; import com.hubspot.singularity.executor.TemplateManager; import com.hubspot.singularity.executor.config.SingularityExecutorConfiguration; -import com.hubspot.singularity.SingularityS3UploaderFile; +import com.hubspot.singularity.executor.config.SingularityExecutorLogrotateAdditionalFile; import com.hubspot.singularity.executor.models.LogrotateCronTemplateContext; import com.hubspot.singularity.executor.models.LogrotateTemplateContext; import com.hubspot.singularity.runner.base.configuration.SingularityRunnerBaseConfiguration; @@ -93,15 +95,50 @@ private boolean writeS3MetadataFileForRotatedFiles(boolean finished) { } private void writeLogrotateFile() { - log.info("Writing logrotate configuration file to {}", getLogrotateConfPath()); + log.info("Writing non-hourly logrotate configuration file to {}", getLogrotateConfPath()); templateManager.writeLogrotateFile(getLogrotateConfPath(), new LogrotateTemplateContext(configuration, taskDefinition)); - if (logrotateFrequency.getCronSchedule().isPresent()) { - log.info("Writing logrotate cron entry with schedule '{}' to {}", logrotateFrequency.getCronSchedule().get(), getLogrotateCronPath()); - templateManager.writeCronEntryForLogrotate(getLogrotateCronPath(), new LogrotateCronTemplateContext(configuration, taskDefinition, logrotateFrequency)); + // Get the frequency and cron schedule for an additional file with an HOURLY schedule, if there is any + Optional additionalFileFrequency = getAdditionalHourlyFileFrequency(); + + // if any additional file or the global setting has an hourly rotation, write a separate rotate config and force rotate using a cron schedule + if (additionalFileFrequency.isPresent() && additionalFileFrequency.get().getCronSchedule().isPresent() || logrotateFrequency.getCronSchedule().isPresent()) { + File hourlyLogrotateDir = new File(configuration.getLogrotateHourlyConfDirectory()); + if (!hourlyLogrotateDir.exists()) { + if (!hourlyLogrotateDir.mkdir()) { + log.warn("Could not create hourly logrotate directory at {}", configuration.getLogrotateHourlyConfDirectory()); + } + } + log.info("Writing hourly logrotate configuration file to {}", getLogrotateHourlyConfPath()); + templateManager.writeHourlyLogrotateFile(getLogrotateHourlyConfPath(), new LogrotateTemplateContext(configuration, taskDefinition)); + + SingularityExecutorLogrotateFrequency freq = additionalFileFrequency.isPresent() && additionalFileFrequency.get().getCronSchedule().isPresent() ? + additionalFileFrequency.get() : logrotateFrequency; + + String cronScheduleString = freq.getCronSchedule().isPresent() ? freq.getCronSchedule().get() : "Error in cron schedule"; // This should never evaluate to false + + log.info("Writing logrotate cron entry with schedule '{}' to {}", cronScheduleString, getLogrotateCronPath()); + templateManager.writeCronEntryForLogrotate(getLogrotateCronPath(), new LogrotateCronTemplateContext(configuration, taskDefinition, freq)); } } + /** + * + * @return Frequency (and contained cron schedule) of the hourly rotation additional file + */ + private Optional getAdditionalHourlyFileFrequency() { + + for (SingularityExecutorLogrotateAdditionalFile file : configuration.getLogrotateAdditionalFiles()) { + if (file.getLogrotateFrequencyOverride().isPresent() && + file.getLogrotateFrequencyOverride().get().equals(SingularityExecutorLogrotateFrequency.HOURLY) && + file.getLogrotateFrequencyOverride().get().getCronSchedule().isPresent()) { + + return Optional.of(file.getLogrotateFrequencyOverride().get()); + } + } + return Optional.absent(); + } + @SuppressFBWarnings public boolean teardown() { boolean writeTailMetadataSuccess = writeTailMetadata(true); @@ -130,7 +167,7 @@ public boolean teardown() { return writeTailMetadataSuccess && removeLogRotateFileSuccess && writeS3MetadataForLogrotatedFilesSuccess && writeS3MetadataForNonLogRotatedFileSuccess; } else { - return false; + return removeLogrotateFile(); } } @@ -162,17 +199,40 @@ private void copyLogTail() { public boolean removeLogrotateFile() { boolean deleted = false; try { - deleted = Files.deleteIfExists(getLogrotateConfPath()); - if (logrotateFrequency.getCronSchedule().isPresent()) { - boolean cronDeleted = Files.deleteIfExists(getLogrotateCronPath()); - deleted = deleted || cronDeleted; + if (Files.exists(getLogrotateConfPath())) { + deleted = Files.deleteIfExists(getLogrotateConfPath()); + log.debug("Deleted {} : {}", getLogrotateConfPath(), deleted); + } else { + deleted = true; + } + } catch (Throwable t){ + log.debug("Couldn't delete {}", getLogrotateConfPath(), t); + return false; + } + + Optional additionalFileFreq = getAdditionalHourlyFileFrequency(); + try { + if ((additionalFileFreq.isPresent() && additionalFileFreq.get().getCronSchedule().isPresent()) || logrotateFrequency.getCronSchedule().isPresent()) { + boolean hourlyConfDeleted = !Files.exists(getLogrotateHourlyConfPath()) || Files.deleteIfExists(getLogrotateHourlyConfPath()); + log.debug("Deleted {} : {}", getLogrotateHourlyConfPath(), deleted); + deleted = deleted && hourlyConfDeleted; } } catch (Throwable t) { - log.trace("Couldn't delete {}", getLogrotateConfPath(), t); + log.debug("Couldn't delete {}", getLogrotateHourlyConfPath(), t); return false; } - log.trace("Deleted {} : {}", getLogrotateConfPath(), deleted); - return true; + + try { + if ((additionalFileFreq.isPresent() && additionalFileFreq.get().getCronSchedule().isPresent()) || logrotateFrequency.getCronSchedule().isPresent()) { + boolean cronDeleted = !Files.exists(getLogrotateCronPath()) || Files.deleteIfExists(getLogrotateCronPath()); + log.debug("Deleted {} : {}", getLogrotateCronPath(), deleted); + deleted = deleted && cronDeleted; + } + } catch (Throwable t) { + log.debug("Couldn't delete {}", getLogrotateCronPath(), t); + return false; + } + return deleted; } public boolean manualLogrotate() { @@ -247,6 +307,10 @@ public Path getLogrotateConfPath() { return Paths.get(configuration.getLogrotateConfDirectory()).resolve(taskDefinition.getTaskId()); } + public Path getLogrotateHourlyConfPath() { + return Paths.get(configuration.getLogrotateHourlyConfDirectory()).resolve(taskDefinition.getTaskId()); + } + public Path getLogrotateCronPath() { return Paths.get(configuration.getCronDirectory()).resolve(taskDefinition.getTaskId() + ".logrotate"); } diff --git a/SingularityExecutor/src/main/resources/logrotate.conf.hbs b/SingularityExecutor/src/main/resources/logrotate.conf.hbs index 2dac6c2de1..6d09ba1571 100644 --- a/SingularityExecutor/src/main/resources/logrotate.conf.hbs +++ b/SingularityExecutor/src/main/resources/logrotate.conf.hbs @@ -12,6 +12,7 @@ rotate {{{ rotateCount }}} maxage {{{ maxageDays }}} notifempty +{{#unless isGlobalLogrotateHourly}} {{#if shouldLogRotateLogFile }} {{{ logfile }}} { dateformat -{{{ rotateDateformat }}}{{#if logfileExtension}}.{{{ logfileExtension}}}{{/if}} @@ -33,9 +34,11 @@ notifempty {{/if}} } {{/if}} +{{/unless}} {{#if extrasFiles}} {{#each extrasFiles}}{{{filename}}} { + {{#if logrotateFrequencyOverride }}{{{logrotateFrequencyOverride}}}{{/if}} dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}} missingok {{#if useFileAttributes}} diff --git a/SingularityExecutor/src/main/resources/logrotate.hourly.conf.hbs b/SingularityExecutor/src/main/resources/logrotate.hourly.conf.hbs new file mode 100644 index 0000000000..321d4409bc --- /dev/null +++ b/SingularityExecutor/src/main/resources/logrotate.hourly.conf.hbs @@ -0,0 +1,93 @@ +dateext +compress +{{#if compressCmd}}compresscmd {{{compressCmd}}}{{/if}} +{{#if uncompressCmd}}uncompresscmd {{{uncompressCmd}}}{{/if}} +{{#if compressOptions}}compressoptions {{{compressOptions}}}{{/if}} +{{#if compressExt}}compressext {{{compressExt}}}{{/if}} +{{{ logrotateFrequency }}} +copytruncate +nomail +nosharedscripts +rotate {{{ rotateCount }}} +maxage {{{ maxageDays }}} +notifempty + +{{#if isGlobalLogrotateHourly}} +{{#if shouldLogRotateLogFile }} +{{{ logfile }}} { + dateformat -{{{ rotateDateformat }}}{{#if logfileExtension}}.{{{ logfileExtension}}}{{/if}} + olddir {{{ rotateDirectory }}} +{{#if useFileAttributes}} + lastaction + NOW="$(($(date +%s%N)/1000000))" + LOGSTART=`getfattr --only-values -n user.logstart {{{ logfile }}}` + timestring=`date +%Y-%m-%d-%s` + for filename in "{{{ taskDirectory }}}/{{{ rotateDirectory }}}/{{{ logfileName }}}-${timestring%???}"* + do + setfattr -n user.logend -v "$NOW" $filename + if [ "$LOGSTART" != "" ]; then + setfattr -n user.logstart -v "$LOGSTART" $filename + fi + done + setfattr -n user.logstart -v "$NOW" {{{ logfile }}} + endscript +{{/if}} +} +{{/if}} + +{{#if extrasFiles}} +{{#each extrasFiles}}{{{filename}}} { + {{#if logrotateFrequencyOverride }}{{{logrotateFrequencyOverride}}}{{/if}} + dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}} + missingok +{{#if useFileAttributes}} + lastaction + NOW="$(($(date +%s%N)/1000000))" + for oldfile in $@ + do + LOGSTART=`getfattr --only-values -n user.logstart $oldfile` + timestring=`date +%Y-%m-%d-%s` + for filename in "$oldfile-${timestring%???}"* + do + if [ "$LOGSTART" != "" ]; then + setfattr -n user.logstart -v "$LOGSTART" $filename + fi + setfattr -n user.logend -v "$NOW" $filename + done + setfattr -n user.logstart -v "$NOW" $oldfile + done + endscript +{{/if}} +} + +{{/each}} +{{/if}} +{{/if}} + +{{#if extrasFilesHourly}} +{{#each extrasFilesHourly}}{{{filename}}} { + {{#if logrotateFrequencyOverride }}{{{logrotateFrequencyOverride}}}{{/if}} + dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}} + missingok +{{#if useFileAttributes}} + lastaction + NOW="$(($(date +%s%N)/1000000))" + for oldfile in $@ + do + LOGSTART=`getfattr --only-values -n user.logstart $oldfile` + timestring=`date +%Y-%m-%d-%s` + for filename in "$oldfile-${timestring%???}"* + do + if [ "$LOGSTART" != "" ]; then + setfattr -n user.logstart -v "$LOGSTART" $filename + fi + setfattr -n user.logend -v "$NOW" $filename + done + setfattr -n user.logstart -v "$NOW" $oldfile + done + endscript +{{/if}} +} + +{{/each}} +{{/if}} diff --git a/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java b/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java index ea865fc2f6..b3415c45ad 100644 --- a/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java +++ b/SingularityExecutor/src/test/java/com/hubspot/singularity/executor/config/SingularityExecutorConfigurationTest.java @@ -1,18 +1,26 @@ package com.hubspot.singularity.executor.config; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import java.io.File; import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import javax.validation.Validator; import org.junit.Test; import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.jknack.handlebars.Handlebars; +import com.github.jknack.handlebars.Template; import com.google.common.base.Optional; import com.google.common.base.Throwables; +import com.hubspot.singularity.executor.SingularityExecutorLogrotateFrequency; +import com.hubspot.singularity.executor.models.LogrotateAdditionalFile; +import com.hubspot.singularity.executor.models.LogrotateTemplateContext; import com.hubspot.singularity.runner.base.config.SingularityRunnerBaseModule; import com.hubspot.singularity.runner.base.config.SingularityRunnerConfigurationProvider; @@ -62,4 +70,48 @@ private SingularityExecutorConfiguration loadConfig(String file) { } } + @Test + public void itProperlyGeneratesTwoLogrotateConfigs() throws Exception { + Handlebars handlebars = new Handlebars(); + Template hourlyTemplate = handlebars.compile(SingularityExecutorModule.LOGROTATE_HOURLY_TEMPLATE); + Template nonHourlyTemplate = handlebars.compile(SingularityExecutorModule.LOGROTATE_TEMPLATE); + + LogrotateTemplateContext context = mock(LogrotateTemplateContext.class); + + List testExtraFiles = new ArrayList<>(); + List testExtraFilesHourly = new ArrayList<>(); + + testExtraFiles.add(new LogrotateAdditionalFile("/tmp/testfile.txt", "txt", "%Y%m%d", + Optional.of(SingularityExecutorLogrotateFrequency.MONTHLY))); + + testExtraFilesHourly.add(new LogrotateAdditionalFile("/tmp/testfile-hourly.txt", "txt", "%Y%m%d", + Optional.of(SingularityExecutorLogrotateFrequency.HOURLY))); + + doReturn(SingularityExecutorLogrotateFrequency.WEEKLY.getLogrotateValue()).when(context).getLogrotateFrequency(); + doReturn(testExtraFiles).when(context).getExtrasFiles(); + doReturn(testExtraFilesHourly).when(context).getExtrasFilesHourly(); + doReturn(false).when(context).isGlobalLogrotateHourly(); + + // This sample output template, when copied into a staged Mesos slave and run with `logrotate -d ` + // confirms that a testfile.txt at the /tmp/testfile.txt will be cycled daily instead of weekly + String hourlyOutput = hourlyTemplate.apply(context); + String nonHourlyOutput = nonHourlyTemplate.apply(context); + + // Assert that our config has both weekly and daily scopes, and that daily occurs second (thus overrides weekly + // in the /tmp/testfile-hourly.txt YAML object). + + // Admittedly, YAML serialization would be better, but given this code doesn't actually test much without + // a binary of `logrotate` to run against, it's not a big deal. + assertThat(hourlyOutput.contains("weekly")).isTrue(); + assertThat(hourlyOutput.contains("daily")).isTrue(); + assertThat(hourlyOutput.indexOf("daily")).isGreaterThan(hourlyOutput.indexOf("weekly")); + + // Assert that our config has both weekly and monthly scopes, and that monthly occurs second (thus overrides weekly + // in the /tmp/testfile.txt YAML object). + assertThat(nonHourlyOutput.contains("weekly")).isTrue(); + assertThat(nonHourlyOutput.contains("monthly")).isTrue(); + assertThat(nonHourlyOutput.indexOf("monthly")).isGreaterThan(hourlyOutput.indexOf("weekly")); + + } + }