Even better date handling

Stumbled into some issues with the date handling in the code base. This make it more usable. * Removes the `--date-separator|--date-sep` CLI switches * Uses a regex to match `YYYY.MM.DD`, `YYYY-MM-DD`, or `YYYYMMDD` in index names to attain the date string of the index
reyjrar · Aug 9, 2023 · 2c88955 · 2c88955
1 parent 4c9fd82
commit 2c88955
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 51 deletions.
diff --git a/CopyIndexes.mkdn b/CopyIndexes.mkdn
@@ -4,7 +4,7 @@ es-copy-index.pl - Copy an index from one cluster to another
 
 # VERSION
 
-version 8.6
+version 8.7
 
 # SYNOPSIS
 
@@ -42,7 +42,6 @@ From App::ElasticSearch::Utilities:
     --index         Index to run commands against
     --base          For daily indexes, reference only those starting with "logstash"
                      (same as --pattern logstash-* or logstash-DATE)
-    --datesep       Date separator, default '.' also (--date-separator)
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: 1
 

diff --git a/Maintenance.mkdn b/Maintenance.mkdn
@@ -4,7 +4,7 @@ es-daily-index-maintenance.pl - Run to prune old indexes and optimize existing
 
 # VERSION
 
-version 8.6
+version 8.7
 
 # SYNOPSIS
 
@@ -46,7 +46,6 @@ From App::ElasticSearch::Utilities:
     --index         Index to run commands against
     --base          For daily indexes, reference only those starting with "logstash"
                      (same as --pattern logstash-* or logstash-DATE)
-    --datesep       Date separator, default '.' also (--date-separator)
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: 1
 

diff --git a/README.mkdn b/README.mkdn
@@ -4,7 +4,7 @@ App::ElasticSearch::Utilities - Utilities for Monitoring ElasticSearch
 
 # VERSION
 
-version 8.6
+version 8.7
 
 # SYNOPSIS
 
@@ -285,7 +285,6 @@ From App::ElasticSearch::Utilities:
     --index         Index to run commands against
     --base          For daily indexes, reference only those starting with "logstash"
                      (same as --pattern logstash-* or logstash-DATE)
-    --datesep       Date separator, default '.' also (--date-separator)
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: 1
 

diff --git a/Searching.mkdn b/Searching.mkdn
@@ -4,7 +4,7 @@ es-search.pl - Provides a CLI for quick searches of data in ElasticSearch daily
 
 # VERSION
 
-version 8.6
+version 8.7
 
 # SYNOPSIS
 
@@ -62,7 +62,6 @@ From App::ElasticSearch::Utilities:
     --index         Index to run commands against
     --base          For daily indexes, reference only those starting with "logstash"
                      (same as --pattern logstash-* or logstash-DATE)
-    --datesep       Date separator, default '.' also (--date-separator)
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: 1
 

diff --git a/lib/App/ElasticSearch/Utilities.pm b/lib/App/ElasticSearch/Utilities.pm
@@ -101,7 +101,6 @@ From App::ElasticSearch::Utilities:
     --index         Index to run commands against
     --base          For daily indexes, reference only those starting with "logstash"
                      (same as --pattern logstash-* or logstash-DATE)
-    --datesep       Date separator, default '.' also (--date-separator)
     --pattern       Use a pattern to operate on the indexes
     --days          If using a pattern or base, how many days back to go, default: 1
 
@@ -249,9 +248,19 @@ The indexes are compared against this pattern.
 # Global Variables
 our %_GLOBALS = ();
 my  %DEF      = ();
-my %PATTERN_REGEX = (
+my  %PATTERN_REGEX = (
     '*'  => qr/.*/,
     ANY  => qr/.*/,
+    DATE => qr/
+        (?<datestr>
+                (?<year>\d{4})              # Extract 4 digits for the year
+                (?:(?<datesep>[\-.]))?      # Optionally, look for . - as a separator
+                (?<month>\d{2})             # Two digits for the month
+                \g{datesep}                 # Whatever the date separator was in the previous match
+                (?<day>\d{2})               # Two digits for the day
+                (?![a-zA-Z0-9])             # Zero width negative look ahead, not alphanumeric
+        )
+    /x,
 );
 my $PATTERN;
 
@@ -283,7 +292,6 @@ my $PATTERN;
             base|index-basename=s
             days=i
             noop!
-            datesep|date-separator=s
             proto=s
             http-username=s
             password-exec=s
@@ -390,10 +398,6 @@ sub es_utils_initialize {
         PATTERN     => exists $opts->{pattern} ? $opts->{pattern} : '*',
         DAYS        => exists $opts->{days}    ? $opts->{days}
                      : exists $_GLOBALS{days}  ? $_GLOBALS{days} : 1,
-        DATESEP     => exists $opts->{datesep}            ? $opts->{datesep}
-                     : exists $_GLOBALS{datesep}          ? $_GLOBALS{datesep}
-                     : exists $_GLOBALS{"date-separator"} ? $_GLOBALS{"date-separator"}
-                     : '.',
         # HTTP Basic Authentication
         USERNAME    => exists $opts->{'http-username'}    ? $opts->{'http-username'}
                      : exists $_GLOBALS{'http-username'}  ? $_GLOBALS{'http-username'}
@@ -425,20 +429,11 @@ sub es_utils_initialize {
         delete $ENV{$_} for qw(http_proxy HTTP_PROXY);
     }
 
-    # Setup Variables based on the config
-    %PATTERN_REGEX = (
-        '*'  => qr/.*/,
-        DATE => qr/\d{4}(?:\Q$DEF{DATESEP}\E)?\d{2}(?:\Q$DEF{DATESEP}\E)?\d{2}/,
-        ANY  => qr/.*/,
-    );
-    my @ordered = qw(* DATE ANY);
-
-    if( index($DEF{DATESEP},'-') >= 0 ) {
-        output({stderr=>1,color=>'yellow'}, "=== Using a '-' as your date separator may cause problems with other utilities. ===");
-    }
 
     # Build the Index Pattern
     $PATTERN = $DEF{PATTERN};
+
+    my @ordered = qw(* DATE ANY);
     foreach my $literal ( @ordered ) {
         $PATTERN =~ s/\Q$literal\E/$PATTERN_REGEX{$literal}/g;
     }
@@ -1043,7 +1038,6 @@ sub es_indices {
                 }
             }
             elsif( $args{check_dates} && defined $DEF{DAYS} ) {
-
                 my $days_old = es_index_days_old( $index );
                 if( !defined $days_old ) {
                     debug({indent=>2,color=>'red'}, "! error locating date in string, skipping !");
@@ -1081,12 +1075,8 @@ sub es_index_strip_date {
     es_utils_initialize() unless keys %DEF;
 
     # Try the Date Pattern
-    if( $index =~ s/[-_]$PATTERN_REGEX{DATE}.*//o ) {
-        return $index;
-    }
-    # Fallback to matching thing-YYYY-MM-DD or thing-YYYY.MM.DD
-    elsif( $index =~ s/[-_]\d{4}([.-])\d{2}\g{1}\d{2}(?:[-_.]\d+)?$// ) {
-        return $index;
+    if( my $base = $index =~ s/[^a-z0-9]+$PATTERN_REGEX{DATE}.*$//rio ) {
+        return $base;
     }
     return;
 }
@@ -1106,6 +1096,8 @@ sub es_index_bases {
 
     # Strip to the base
     my $stripped = es_index_strip_date($index);
+    # Remove the rollover portion
+    $stripped =~ s/[\-_.]\d+$//;
     return unless defined $stripped and length $stripped;
 
     # Compute if we haven't already memoized
@@ -1132,32 +1124,39 @@ Return the number of days old this index is.
 
 =cut
 
-my $NOW = timegm(0,0,0,(gmtime)[3,4,5]);
 sub es_index_days_old {
     my ($index) = @_;
 
     return unless defined $index;
 
     es_utils_initialize() unless keys %DEF;
 
-    if( my ($dateStr) = ($index =~ /($PATTERN_REGEX{DATE})/) ) {
-        my @date=();
-        if(length $DEF{DATESEP}) {
-           @date = reverse map { int } split /\Q$DEF{DATESEP}\E/, $dateStr;
-        }
-        else {
-            for my $len (qw(4 2 2)) {
-                unshift @date, substr($dateStr,0,$len,'');
-            }
-        }
+
+    if( $index =~ /[^a-z0-9]$PATTERN_REGEX{DATE}/io ) {
+        # Build Date Array
+        my @date = map { int }
+                    grep { length }
+                    map { $+{$_} =~ s/^0//r } qw(day month year);
         $date[1]--; # move 1-12 -> 0-11
+        # Validate
+        if( @date != 3 ) {
+            warn sprintf "es_index_days_old(%s) matched DATE(%s), but did not receive enough parts: %s",
+                $index,
+                $+{datestr},
+                join(', ', map { "'$_'" } @date);
+            return;
+        }
+
+        # Calculate Difference
+        my $now = timegm(0,0,0,(gmtime)[3,4,5]);
         my $idx_time = eval { timegm( 0,0,0, @date ) };
         return unless $idx_time;
-        my $diff = $NOW - $idx_time;
+        my $diff = $now - $idx_time;
         $diff++;    # Add one second
         debug({color=>"yellow"}, sprintf "es_index_days_old(%s) - Time difference is %0.3f", $index, $diff/86400);
         return int($diff / 86400);
     }
+    verbose({color=>"red"}, "es_index_days_old($index) - date string not found");
     return;
 }
 
@@ -1570,9 +1569,9 @@ sub es_local_index_meta {
     es_utils_initialize() unless keys %DEF;
 
     if( exists $_GLOBALS{meta} ) {
-        my $meta = $_GLOBALS{meta};
+        my $meta   = $_GLOBALS{meta};
         my @search = ( $name_or_base );
-        push @search, es_index_strip_date( $name_or_base );
+        push @search, es_index_strip_date($name_or_base);
         push @search, es_index_bases($name_or_base);
 
         foreach my $check ( @search ) {

diff --git a/scripts/es-daily-index-maintenance.pl b/scripts/es-daily-index-maintenance.pl
@@ -28,7 +28,6 @@
     'optimize',
     'optimize-days=i',
     'index-basename=s',
-    'date-separator=s',
     'timezone=s',
     'skip-alias=s@',
     # Basic options

diff --git a/scripts/es-graphite-dynamic.pl b/scripts/es-graphite-dynamic.pl
@@ -73,7 +73,6 @@
 
 #------------------------------------------------------------------------#
 # Collect and Decode the Cluster Statistics
-
 my @metrics = sort map { "$_->{key} $_->{value}" } @{ $Fetcher->get_metrics };
 if( !@metrics ) {
     output({color=>'red'}, "Error retrieving metrics");

diff --git a/t/02-index-data.t b/t/02-index-data.t
@@ -16,7 +16,18 @@ $Data::Dumper::Sortkeys = 1;
 my $now = DateTime->now();
 my @days_old = qw(0 1 3 5 8 13 21 90);
 
-my %TESTS=();
+my %TESTS=(
+    'notadate-100000' => {
+            es_index_bases      => 'notadate',
+            es_index_days_old   => undef,
+            es_index_strip_date => 'notadate-100000',
+    },
+    strftime('mystery-science-theater-3000-%Y.%m.%d', localtime) => {
+            es_index_bases      => 'mystery,mystery-science,mystery-science-theater',
+            es_index_days_old   => 0,
+            es_index_strip_date => 'mystery-science-theater-3000',
+    },
+);
 foreach my $days_old ( @days_old ) {
     # Query String Parser Testing
     my $lt = $now->clone->subtract( days => $days_old );
@@ -37,6 +48,11 @@ foreach my $days_old ( @days_old ) {
             es_index_days_old   => $days_old,
             es_index_strip_date => 'type_dcid',
         },
+        "type_dcid_$date-0001" => {
+            es_index_bases      => 'type,type_dcid',
+            es_index_days_old   => $days_old,
+            es_index_strip_date => 'type_dcid',
+        },
     );
     # Install the test globally
     foreach my $t (keys %tests) {
@@ -50,7 +66,7 @@ foreach my $t (sort keys %TESTS) {
     my $got = {
         es_index_bases      => join(',', es_index_bases($t)),
         es_index_strip_date => es_index_strip_date($t),
-        es_index_days_old   => es_index_days_old($t),
+        es_index_days_old   => es_index_days_old($t) // undef,
     };
     is_deeply($got,$TESTS{$t},sprintf "%s - %s", $t, join(',', sort keys %{$got})) or diag( Dumper $got );
 }