Preparing 8.6 Release

* Add `App::ElasticSearch::Utilities::Metrics`
    * Convert es-graphite-dynamic.pl to use App::ElasticSearch::Utilities::Metrics
* Updates for `App::ElasticSearch::Utilities::Aggregations`:
    * Adding tests for the `es_flatten_aggs()` function
    * For aggs with `key_as_string`, don't lose data by adding `$field.raw` as the `key` value
* New Utilities:
    * `es-index-fields.pl` shows storage details about the fields in indexes
    * `es-index-scan.pl` scans indexes for potential issues
* Updates for `App::ElasticSearch::Utilities`
    * `es_index_strip_date()` - Add a fall back parser for date patterns in index names
    * **(new)** `es_human_count()` to transform 10_000 into "10 thousand"
    * **(new)** `es_human_size()` to transform 10_000 into "10 Kb"
* Require Perl 5.16+, and test on 5.38

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-latest']
-        perl: [ '5.36', '5.34', '5.32', '5.30', '5.28', '5.26', '5.24', '5.22', '5.20', '5.18', '5.16', '5.14' ]
+        perl: [ '5.38', '5.36', '5.34', '5.32', '5.30', '5.28', '5.26', '5.24', '5.22', '5.20', '5.18', '5.16' ]
     name: Perl ${{ matrix.perl }} on ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v3

CopyIndexes.mkdn

@@ -4,7 +4,7 @@ es-copy-index.pl - Copy an index from one cluster to another
 
 # VERSION
 
-version 8.5
+version 8.6
 
 # SYNOPSIS
 

Maintenance.mkdn

@@ -4,7 +4,7 @@ es-daily-index-maintenance.pl - Run to prune old indexes and optimize existing
 
 # VERSION
 
-version 8.5
+version 8.6
 
 # SYNOPSIS
 

README.mkdn

@@ -4,7 +4,7 @@ App::ElasticSearch::Utilities - Utilities for Monitoring ElasticSearch
 
 # VERSION
 
-version 8.5
+version 8.6
 
 # SYNOPSIS
 
@@ -217,6 +217,21 @@ Returns a hashref
 
 Performs flattening that's compatible with Elasticsearch's flattening.
 
+## es\_human\_count
+
+Takes a number and returns the number as a string in docs, thousands, millions, or billions.
+
+    1_000     -> "1.00 thousand",
+    1_000_000 -> "1.00 million",
+
+## es\_human\_size
+
+Takes a number and returns the number as a string in bytes, Kb, Mb, Gb, or Tb using base 1024.
+
+    1024        -> '1.00 Kb',
+    1048576     -> '1.00 Mb',
+    1073741824  -> '1.00 Gb',
+
 ## def('key')
 
 Exposes Definitions grabbed by options parsing

Searching.mkdn

@@ -4,7 +4,7 @@ es-search.pl - Provides a CLI for quick searches of data in ElasticSearch daily
 
 # VERSION
 
-version 8.5
+version 8.6
 
 # SYNOPSIS
 

cpanfile

@@ -42,7 +42,7 @@ requires "YAML::XS" => "0";
 requires "feature" => "0";
 requires "namespace::autoclean" => "0";
 requires "parent" => "0";
-requires "perl" => "5.013002";
+requires "perl" => "v5.16.0";
 requires "strict" => "0";
 requires "version" => "0";
 requires "warnings" => "0";
@@ -54,12 +54,12 @@ on 'test' => sub {
   requires "IO::Handle" => "0";
   requires "IPC::Open3" => "0";
   requires "Test::More" => "0";
-  requires "perl" => "5.013002";
+  requires "perl" => "v5.16.0";
 };
 
 on 'configure' => sub {
   requires "ExtUtils::MakeMaker" => "0";
-  requires "perl" => "5.013002";
+  requires "perl" => "v5.16.0";
 };
 
 on 'develop' => sub {

lib/App/ElasticSearch/Utilities.pm

@@ -1,8 +1,7 @@
 # ABSTRACT: Utilities for Monitoring ElasticSearch
 package App::ElasticSearch::Utilities;
 
-use v5.10;
-use strict;
+use v5.16;
 use warnings;
 
 # VERSION
@@ -62,10 +61,13 @@ use Sub::Exporter -setup => {
         es_apply_index_settings
         es_local_index_meta
         es_flatten_hash
+        es_human_count
+        es_human_size
     )],
     groups => {
         config  => [qw(es_utils_initialize es_globals)],
         default => [qw(es_utils_initialize es_connect es_indices es_request)],
+        human   => [qw(es_human_count es_human_size)],
         indices => [qw(:default es_indices_meta)],
         index   => [qw(:default es_index_valid es_index_fields es_index_days_old es_index_bases)],
     },
@@ -1078,7 +1080,12 @@ sub es_index_strip_date {
 
     es_utils_initialize() unless keys %DEF;
 
-    if( $index =~ s/[-_]$PATTERN_REGEX{DATE}.*// ) {
+    # Try the Date Pattern
+    if( $index =~ s/[-_]$PATTERN_REGEX{DATE}.*//o ) {
+        return $index;
+    }
+    # Fallback to matching thing-YYYY-MM-DD or thing-YYYY.MM.DD
+    elsif( $index =~ s/[-_]\d{4}([.-])\d{2}\g{1}\d{2}(?:[-_.]\d+)?$// ) {
         return $index;
     }
     return;
@@ -1465,6 +1472,53 @@ sub es_flatten_hash {
     return \%compat;
 }
 
+=func es_human_count
+
+Takes a number and returns the number as a string in docs, thousands, millions, or billions.
+
+    1_000     -> "1.00 thousand",
+    1_000_000 -> "1.00 million",
+
+=cut
+
+sub es_human_count {
+    my ($size) = @_;
+
+    my $unit = 'docs';
+    my @units = qw(thousand million billion);
+
+    while( $size > 1000 && @units ) {
+        $size /= 1000;
+        $unit = shift @units;
+    }
+
+    return sprintf "%0.2f %s", $size, $unit;
+}
+
+=func es_human_size
+
+Takes a number and returns the number as a string in bytes, Kb, Mb, Gb, or Tb using base 1024.
+
+    1024        -> '1.00 Kb',
+    1048576     -> '1.00 Mb',
+    1073741824  -> '1.00 Gb',
+
+=cut
+
+sub es_human_size {
+    my ($size) = @_;
+
+    my $unit = 'b';
+    my @units = qw(Kb Mb Gb Tb);
+
+    while( $size > 1024 && @units ) {
+        $size /= 1024;
+        $unit = shift @units;
+    }
+
+    return sprintf "%0.2f %s", $size, $unit;
+}
+
 =func def('key')
 
 Exposes Definitions grabbed by options parsing

lib/App/ElasticSearch/Utilities/Aggregations.pm

@@ -1,7 +1,7 @@
 package App::ElasticSearch::Utilities::Aggregations;
 # ABSTRACT: Code to simplify creating and working with Elasticsearch aggregations
 
-use strict;
+use v5.16;
 use warnings;
 
 use Storable qw(dclone);
@@ -539,6 +539,7 @@ sub es_flatten_aggregations {
             my $k = delete $result->{key};
             my $ks = delete $result->{key_as_string};
             push @{ $row }, $field, $ks || $k;
+            push @{ $row }, "$field.raw", $k if $ks;
             push @{ $row }, "$field.hits", delete $result->{doc_count} || 0;
         }
         my %buckets = ();

lib/App/ElasticSearch/Utilities/Connection.pm

@@ -29,7 +29,7 @@ the API you'd expect from B<Elastijk>.
 
 =cut
 
-use strict;
+use v5.16;
 use warnings;
 
 # VERSION

lib/App/ElasticSearch/Utilities/HTTPRequest.pm

@@ -9,7 +9,7 @@ use this module in your code.
 
 =cut
 
-use strict;
+use v5.16;
 use warnings;
 no warnings 'uninitialized';