Skip to content

Commit

Permalink
rest: add calc and s transformation function
Browse files Browse the repository at this point in the history
  • Loading branch information
@sni
sni committed Apr 23, 2024
1 parent 5624a20 commit fe64d84
Show file tree
Hide file tree
Showing 4 changed files with 137 additions and 16 deletions.
1 change: 1 addition & 0 deletions MANIFEST
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2584,6 +2584,7 @@ t/scenarios/rest_api/t/301-controller_rest_reports.t
t/scenarios/rest_api/t/301-controller_rest_scenario.t
t/scenarios/rest_api/t/305-controller_rest_commands.t
t/scenarios/rest_api/t/local/rest.t
t/scenarios/rest_api/t/local/rest_api_misc.t
t/scenarios/rest_api/t/local/rest_check_thruk_rest.t
t/scenarios/rest_api/t/local/rest_commands.t
t/scenarios/rest_api/t/local/rest_configtool.t
Expand Down
3 changes: 3 additions & 0 deletions docs/documentation/rest.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,9 @@ Available transformations functions are:
* `lower`: make column lower case
* `lc`: alias for lower
* `substr`: extract substring (substr(columnname, offset [, length]))
* `calc`: apply mathematical operation (calc(column, op , value) ex.: calc(last_check, '*', 1000))
* `s`: apply regex replacement (s(column, regex, replace) ex.: s(host_name, '/\..*$/', '') Note: you cannot use backreferences like $1 in the replacement string for security reasons.
* `unit`: set the unit for this column (unit(column, unit) ex.: unit(calc(rta, "*", 1000), "s")

ex.: return first 3 characters from upper case host name.

Expand Down
111 changes: 95 additions & 16 deletions lib/Thruk/Controller/rest_v1.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,11 @@ sub process_rest_request {
}
}

my $wrapped;
$wrapped = 1 if($c->req->header("x-thruk-outputformat") && $c->req->header("x-thruk-outputformat") eq 'wrapped_json');
$wrapped = 1 if(defined $c->req->parameters->{'headers'} && $c->req->parameters->{'headers'} eq 'wrapped_json');
$c->stash->{'meta_column_info'} = {};

if(!$data) {
eval {
$data = _fetch($c, $path_info, $method);
Expand All @@ -207,9 +212,6 @@ sub process_rest_request {
}

# add columns meta data, ex.: used in the thruk grafana datasource
my $wrapped;
$wrapped = 1 if($c->req->header("x-thruk-outputformat") && $c->req->header("x-thruk-outputformat") eq 'wrapped_json');
$wrapped = 1 if(defined $c->req->parameters->{'headers'} && $c->req->parameters->{'headers'} eq 'wrapped_json');
if($wrapped) {
# wrap data along with column meta data
my $request_method = $method || $c->req->method;
Expand Down Expand Up @@ -785,7 +787,7 @@ sub get_request_columns {
if($_ =~ m/\s+as\s+([^\s]+)\s*$/mx) {
$_ =~ s/^.*?\s+as\s+([^\s]+)\s*$/$1/mx; # strip "as alias" from column
} else {
$_ =~ s/^[^:]+:(.*?)$/$1/gmx; # strip alias
$_ =~ s/^[^:]+:([^"']*?)$/$1/gmx; # strip alias
}
}
}
Expand All @@ -794,7 +796,7 @@ sub get_request_columns {
if($_ =~ m/\s+as\s+([^\s]+)\s*$/mx) {
$_ =~ s/\s+as\s+([^\s]+)\s*$//mx; # strip "as alias" from column
} else {
$_ =~ s/^([^:]+):.*?$/$1/gmx; # strip alias
$_ =~ s/^([^:]+):[^"']*?$/$1/gmx; # strip alias
}
$_ =~ s/^.*\(([^)]+)\)$/$1/gmx; # extract column name from aggregation function
}
Expand All @@ -821,7 +823,7 @@ sub get_aliased_columns {
if($col =~ m/^([^:]+)\s+as\s+(.*)$/mx) {
$alias_columns->{$2} = $1;
}
elsif($col =~ m/^([^:]+):(.*)$/mx) {
elsif($col =~ m/^([^:]+):([^"']*)$/mx) {
$alias_columns->{$2} = $1;
}
}
Expand Down Expand Up @@ -929,15 +931,27 @@ sub _apply_columns {
}
}

# extract helpful meta information from data
my $firstrow;
if($data && $data->[0]) {
$firstrow = $data->[0];
}
for my $col (@{$columns}) {
$c->stash->{'meta_column_info'}->{$col->{'orig'}} = $col;
if($firstrow && $firstrow->{$col->{'column'}."_unit"}) {
$col->{'unit'} = $firstrow->{$col->{'column'}."_unit"};
}
}

my $res = [];
for my $d (@{$data}) {
my $row = {};
for my $c (@{$columns}) {
my $val = $d->{$c->{'orig'}} // $d->{$c->{'column'}};
for my $f (@{$c->{'func'}}) {
$val = _apply_data_function($f, $val);
for my $col (@{$columns}) {
my $val = $d->{$col->{'orig'}} // $d->{$col->{'column'}};
for my $f (@{$col->{'func'}}) {
$val = _apply_data_function($col, $f, $val);
}
$row->{$c->{'alias'}} = $val;
$row->{$col->{'alias'}} = $val;
}
push @{$res}, $row;
}
Expand All @@ -947,7 +961,7 @@ sub _apply_columns {

##########################################################
sub _apply_data_function {
my($f, $val) = @_;
my($col, $f, $val) = @_;
my($name, $args) = @{$f};
if($name eq 'lower' || $name eq 'lc') {
return lc($val // '');
Expand All @@ -962,7 +976,41 @@ sub _apply_data_function {
if(defined $args->[0]) {
return(substr($val // '', $args->[0]));
}
die("usage: substr(value, start[, length])");
die("usage: substr(column, start[, length])");
}
if($name eq 'calc') {
my $op = _trim_quotes($args->[0]);
my $v = _trim_quotes($args->[1]);
die("usage: calc(column, 'operator', 'value')") if(!defined $op || !defined $v);
if($op eq '*') {
$val *= $v;
} elsif($op eq '/') {
$val = $val / $v;
} elsif($op eq '+' || $op eq ' ' || $op eq '') { # assume space as +, might have been url decoded
$val = $val + $v;
} elsif($op eq '-') {
$val = $val - $v;
} else {
die("unsupported operator, use one of: + - * /");
}
return $val;
}
if($name eq 's') {
my $regexp = _trim_quotes($args->[0]);
my $replace = _trim_quotes($args->[1]);
die("usage: s(column, 'regexp', 'replacement')") if(!$regexp || !defined $replace);
$regexp = _trim_re($regexp);
## no critic
my $re = qr($regexp);
$val =~ s/$re/$replace/g;
## use critic
return $val;
}

# just set the unit, do not change the value
if($name eq 'unit') {
$col->{'unit'} = _trim_quotes($args->[0]);
return $val;
}

die("unknown function: ".$name);
Expand Down Expand Up @@ -1315,14 +1363,20 @@ sub _get_columns_meta_for_path {
last;
}

return unless scalar keys %{$columns} > 0;

my $meta = [];
for my $d (@{$req_columns}) {
my $col = $columns->{$alias_columns->{$d} // $d} // {};
$col->{'name'} = $d;
my $hint = $c->stash->{'meta_column_info'}->{$alias_columns->{$d} // $d} // $c->stash->{'meta_column_info'}->{$d};
if((!defined $col->{'config'} || !defined $col->{'config'}->{'unit'}) && $hint) {
if(defined $hint->{'unit'}) {
$col->{'config'}->{'unit'} = $hint->{'unit'};
$col->{'type'} = 'number' unless $col->{'type'};
}
}
push @{$meta}, $col;
}

return $meta;
}

Expand Down Expand Up @@ -2498,7 +2552,7 @@ sub _parse_columns_data {
if($c =~ m/^(.+)\s+as\s+(.*?)$/gmx) {
$name = $1;
$alias = $2;
} elsif($c =~ m/^(.+):(.*?)$/gmx) {
} elsif($c =~ m/^(.+):([^"']*?)$/gmx) {
$name = $1;
$alias = $2;
}
Expand Down Expand Up @@ -2586,6 +2640,31 @@ sub _split_by_comma {
return @res;
}

##########################################################
# remove quotes from string
sub _trim_quotes {
my($val) = @_;
return unless defined $val;
if($val =~ s/^'(.*)'$/$1/mx) {
return $val;
}
if($val =~ s/^"(.*)"$/$1/mx) {
return $val;
}
return $val;
}

##########################################################
# remove slashes from string
sub _trim_re {
my($val) = @_;
return unless defined $val;
if($val =~ s|^/(.*)/$|$1|mx) {
return $val;
}
return $val;
}

##########################################################

1;
38 changes: 38 additions & 0 deletions t/scenarios/rest_api/t/local/rest_api_misc.t
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
use warnings;
use strict;
use Test::More;
use utf8;

BEGIN {
use lib('t');
require TestUtils;
import TestUtils;
}
plan tests => 20;

###########################################################
# test thruks script path
TestUtils::test_command({
cmd => '/bin/bash -c "type thruk"',
like => ['/\/thruk\/script\/thruk/'],
}) or BAIL_OUT("wrong thruk path");

$ENV{'THRUK_TEST_AUTH_KEY'} = "testkey";
$ENV{'THRUK_TEST_AUTH_USER'} = "omdadmin";

###########################################################
# rest api text transformation
{
TestUtils::test_command({
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=name,calc(rta, "+", 1) as rta_plus&headers=wrapped_json\'',
like => ['/rta_plus/', '/localhost/', '/"ms"/'],
});
TestUtils::test_command({
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=name,unit(calc(rta, "*", 1000), "s") as rta_seconds&headers=wrapped_json\'',
like => ['/rta_seconds/', '/localhost/', '/"s"/'],
});
TestUtils::test_command({
cmd => '/usr/bin/env thruk r \'/hosts/localhost?columns=substr(name,0,3)\'',
like => ['/"loc"/'],
});
};

0 comments on commit fe64d84

Please sign in to comment.