Only run force logrotates for cron-faked logrotate confs which actually exist on the filesystem

SingularityBase/src/main/java/com/hubspot/singularity/SingularityS3UploaderFile.java

@@ -151,6 +151,8 @@ public String toString() {
       applyS3StorageClassAfterBytes +
       ", checkSubdirectories=" +
       checkSubdirectories +
+      ", compressBeforeUpload=" +
+      compressBeforeUpload +
       '}'
     );
   }

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateAdditionalFile.java

@@ -36,10 +36,14 @@ public String getDateformat() {
     return dateformat;
   }
 
-  public String getLogrotateFrequencyOverride() {
-    return logrotateFrequencyOverride.isPresent()
-      ? logrotateFrequencyOverride.get().getLogrotateValue()
-      : "";
+  public Optional<SingularityExecutorLogrotateFrequency> getLogrotateFrequencyOverride() {
+    return logrotateFrequencyOverride;
+  }
+
+  public String getLogrotateFrequencyOverrideValue() {
+    return logrotateFrequencyOverride
+      .map(SingularityExecutorLogrotateFrequency::getLogrotateValue)
+      .orElse("");
   }
 
   public String getLogrotateSizeOverride() {

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/models/LogrotateTemplateContext.java

@@ -19,11 +19,10 @@
  */
 public class LogrotateTemplateContext {
   private static final Predicate<LogrotateAdditionalFile> BELONGS_IN_HOURLY_OR_MORE_FREQUENT_CRON_FORCED_LOGROTATE_CONF = p ->
-    SingularityExecutorLogrotateFrequency
-      .HOURLY_OR_MORE_FREQUENT_LOGROTATE_VALUES.stream()
-      .map(SingularityExecutorLogrotateFrequency::getLogrotateValue)
-      .collect(Collectors.toSet())
-      .contains(p.getLogrotateFrequencyOverride());
+    p.getLogrotateFrequencyOverride().isPresent() &&
+    SingularityExecutorLogrotateFrequency.HOURLY_OR_MORE_FREQUENT_LOGROTATE_VALUES.contains(
+      p.getLogrotateFrequencyOverride().get()
+    );
 
   private static final Predicate<LogrotateAdditionalFile> BELONGS_IN_SIZE_BASED_LOGROTATE_CONF = p ->
     p.getLogrotateSizeOverride() != null && !p.getLogrotateSizeOverride().isEmpty();
@@ -113,9 +112,12 @@ public List<LogrotateAdditionalFile> getExtrasFiles() {
   }
 
   /**
-   * Extra files for logrotate to rotate hourly or .
-   * Since we don't want to rely on native `hourly` (or more frequent) support in logrotate(8), we fake it by running an hourly cron with a force `-f` flag.
+   * Extra files for logrotate to rotate hourly or more frequently than hourly.
+   * Since we don't want to rely on native `hourly` (or more frequent) support in logrotate(8),
+   *   we fake it by running an hourly cron with a force `-f` flag.
    * If these do not exist logrotate will continue without error.
+   * If `setExtrasFilesFrequencyFilter()` has been called on this instance,
+   *   then we only return matching logrotateAdditionalFiles configs.
    * @return filenames to rotate.
    */
   public List<LogrotateAdditionalFile> getExtrasFilesHourlyOrMoreFrequent() {
@@ -131,7 +133,11 @@ public List<LogrotateAdditionalFile> getExtrasFilesHourlyOrMoreFrequent() {
               file ->
                 file
                   .getLogrotateFrequencyOverride()
-                  .equals(singularityExecutorLogrotateFrequency.getLogrotateValue())
+                  .map(
+                    someFrequencyOverride ->
+                      someFrequencyOverride == singularityExecutorLogrotateFrequency
+                  )
+                  .orElse(false)
             )
             .collect(Collectors.toList())
       )

SingularityExecutor/src/main/java/com/hubspot/singularity/executor/task/SingularityExecutorTaskLogManager.java

@@ -531,11 +531,10 @@ public boolean manualLogrotate() {
     Set<Path> frequencyBasedPaths = getCronFakedLogrotateAdditionalFileFrequencies()
       .stream()
       .map(this::getLogrotateHourlyConfPath)
+      .filter(Files::exists)
       .collect(Collectors.toSet());
     boolean regularConfExists = Files.exists(getLogrotateConfPath());
-    boolean hourlyConfExists = frequencyBasedPaths
-      .stream()
-      .anyMatch(p -> Files.exists(p));
+    boolean hourlyConfExists = !frequencyBasedPaths.isEmpty();
     boolean sizeConfExists = Files.exists(getLogrotateSizeBasedConfPath());
     if (!sizeConfExists && !hourlyConfExists && !regularConfExists) {
       log.info(

SingularityExecutor/src/main/resources/logrotate.conf.hbs

@@ -38,7 +38,7 @@ notifempty
 
 {{#if extrasFiles}}
 {{#each extrasFiles}}{{{filename}}} {
-  {{#if logrotateFrequencyOverride }}{{{logrotateFrequencyOverride}}}{{/if}}
+  {{#if logrotateFrequencyOverrideValue }}{{{logrotateFrequencyOverrideValue}}}{{/if}}
   dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}}
   missingok
 {{#if useFileAttributes}}

SingularityExecutor/src/main/resources/logrotate.hourlyormorefrequent.conf.hbs

@@ -40,8 +40,8 @@ notifempty
   {{#if logrotateSizeOverride }}
     size {{{logrotateSizeOverride}}}
   {{else}}
-    {{#if logrotateFrequencyOverride }}
-      {{{logrotateFrequencyOverride}}}
+    {{#if logrotateFrequencyOverrideValue }}
+      {{{logrotateFrequencyOverrideValue}}}
     {{/if}}
   {{/if}}
   dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}}
@@ -75,8 +75,8 @@ notifempty
   {{#if logrotateSizeOverride }}
     size {{{logrotateSizeOverride}}}
   {{else}}
-    {{#if logrotateFrequencyOverride }}
-      {{{logrotateFrequencyOverride}}}
+    {{#if logrotateFrequencyOverrideValue }}
+      {{{logrotateFrequencyOverrideValue}}}
     {{/if}}
   {{/if}}
   dateformat -{{{ dateformat }}}{{#if extension}}.{{{ extension}}}{{/if}}

SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/S3UploadMetadata.java

@@ -275,6 +275,8 @@ public String toString() {
       uploadImmediately +
       ", checkSubdirectories=" +
       checkSubdirectories +
+      ", compressBeforeUpload=" +
+      compressBeforeUpload +
       ", uploaderType=" +
       uploaderType +
       ", gcsStorageClass=" +

SingularityRunnerBase/src/main/java/com/hubspot/singularity/runner/base/shared/SimpleProcessManager.java

@@ -136,9 +136,10 @@ public List<String> runCommand(
     if (exitCode.isPresent() && !acceptableExitCodes.contains(exitCode.get())) {
       throw new ProcessFailedException(
         String.format(
-          "Got unacceptable exit code %s while running %s",
+          "Got unacceptable exit code %s while running %s. %s",
           exitCode,
-          processToString
+          processToString,
+          reader.isPresent() ? "Output was " + String.join("\n", reader.get().output) : ""
         )
       );
     }