Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor action messaging to report on No Data state #64365

Merged
merged 16 commits into from
Apr 30, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import {
EuiText,
EuiFormRow,
EuiButtonEmpty,
EuiCheckbox,
EuiToolTip,
EuiIcon,
EuiFieldSearch,
} from '@elastic/eui';
import { IFieldType } from 'src/plugins/data/public';
Expand Down Expand Up @@ -57,6 +60,7 @@ interface Props {
groupBy?: string;
filterQuery?: string;
sourceId?: string;
alertOnNoData?: boolean;
};
alertsContext: AlertsContextValue<AlertContextMeta>;
setAlertParams(key: string, value: any): void;
Expand Down Expand Up @@ -282,6 +286,28 @@ export const Expressions: React.FC<Props> = props => {
</EuiButtonEmpty>
</div>

<EuiSpacer size={'m'} />
<EuiCheckbox
id="metrics-alert-no-data-toggle"
label={
<>
{i18n.translate('xpack.infra.metrics.alertFlyout.alertOnNoData', {
defaultMessage: "Alert me if there's no data",
})}{' '}
<EuiToolTip
content={i18n.translate('xpack.infra.metrics.alertFlyout.noDataHelpText', {
defaultMessage:
'Enable this to trigger the action if the metric(s) do not report any data over the expected time period, or if the alert fails to query Elasticsearch',
})}
>
<EuiIcon type="questionInCircle" color="subdued" />
</EuiToolTip>
</>
}
checked={alertParams.alertOnNoData}
onChange={e => setAlertParams('alertOnNoData', e.target.checked)}
/>

<EuiSpacer size={'m'} />

<EuiFormRow
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ export function getAlertType(): AlertTypeModel {
defaultActionMessage: i18n.translate(
'xpack.infra.metrics.alerting.threshold.defaultActionMessage',
{
defaultMessage: `\\{\\{alertName\\}\\} - \\{\\{context.group\\}\\}
defaultMessage: `\\{\\{alertName\\}\\} - \\{\\{context.group\\}\\} is in a state of \\{\\{context.alertState\\}\\}

\\{\\{context.metricOf.condition0\\}\\} has crossed a threshold of \\{\\{context.thresholdOf.condition0\\}\\}
Current value is \\{\\{context.valueOf.condition0\\}\\}
Reason:
phillipb marked this conversation as resolved.
Show resolved Hide resolved
\\{\\{context.reason\\}\\}
`,
}
),
Expand Down
109 changes: 109 additions & 0 deletions x-pack/plugins/infra/server/lib/alerting/metric_threshold/messages.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { i18n } from '@kbn/i18n';
import { Comparator, AlertStates } from './types';

export const DOCUMENT_COUNT_I18N = i18n.translate(
'xpack.infra.metrics.alerting.threshold.documentCount',
{
defaultMessage: 'Document count',
}
);

export const stateToAlertMessage = {
[AlertStates.ALERT]: i18n.translate('xpack.infra.metrics.alerting.threshold.alertState', {
defaultMessage: 'ALERT',
}),
[AlertStates.NO_DATA]: i18n.translate('xpack.infra.metrics.alerting.threshold.noDataState', {
defaultMessage: 'NO DATA',
}),
[AlertStates.ERROR]: i18n.translate('xpack.infra.metrics.alerting.threshold.errorState', {
defaultMessage: 'ERROR',
}),
// TODO: Implement recovered message state
[AlertStates.OK]: i18n.translate('xpack.infra.metrics.alerting.threshold.okState', {
defaultMessage: 'OK [Recovered]',
}),
};

const comparatorToI18n = (comparator: Comparator, threshold: number[], currentValue: number) => {
const gtText = i18n.translate('xpack.infra.metrics.alerting.threshold.gtComparator', {
defaultMessage: 'greater than',
});
const ltText = i18n.translate('xpack.infra.metrics.alerting.threshold.ltComparator', {
defaultMessage: 'less than',
});
const eqText = i18n.translate('xpack.infra.metrics.alerting.threshold.eqComparator', {
defaultMessage: 'equal to',
});

switch (comparator) {
case Comparator.BETWEEN:
return i18n.translate('xpack.infra.metrics.alerting.threshold.betweenComparator', {
defaultMessage: 'between',
});
case Comparator.OUTSIDE_RANGE:
return i18n.translate('xpack.infra.metrics.alerting.threshold.outsideRangeComparator', {
defaultMessage: 'not between',
});
case Comparator.GT:
return gtText;
case Comparator.LT:
return ltText;
case Comparator.GT_OR_EQ:
case Comparator.LT_OR_EQ:
if (threshold[0] === currentValue) return eqText;
else if (threshold[0] < currentValue) return ltText;
return gtText;
}
};

const thresholdToI18n = ([a, b]: number[]) => {
if (typeof b === 'undefined') return a;
return i18n.translate('xpack.infra.metrics.alerting.threshold.thresholdRange', {
defaultMessage: '{a} and {b}',
values: { a, b },
});
};

export const buildFiredAlertReason: (alertResult: {
metric: string;
comparator: Comparator;
threshold: number[];
currentValue: number;
}) => string = ({ metric, comparator, threshold, currentValue }) =>
i18n.translate('xpack.infra.metrics.alerting.threshold.firedAlertReason', {
defaultMessage:
'{metric} is {comparator} a threshold of {threshold} (current value is {currentValue})',
values: {
metric,
comparator: comparatorToI18n(comparator, threshold, currentValue),
threshold: thresholdToI18n(threshold),
currentValue,
},
});

export const buildNoDataAlertReason: (alertResult: {
metric: string;
timeSize: number;
timeUnit: string;
}) => string = ({ metric, timeSize, timeUnit }) =>
i18n.translate('xpack.infra.metrics.alerting.threshold.noDataAlertReason', {
defaultMessage: '{metric} has reported no data over the past {interval}',
values: {
metric,
interval: `${timeSize}${timeUnit}`,
},
});

export const buildErrorAlertReason = (metric: string) =>
i18n.translate('xpack.infra.metrics.alerting.threshold.errorAlertReason', {
defaultMessage: 'Elasticsearch failed when attempting to query data for {metric}',
values: {
metric,
},
});
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ services.callCluster.mockImplementation(async (_: string, { body, index }: any)
}
if (metric === 'test.metric.2') {
return mocks.alternateMetricResponse;
} else if (metric === 'test.metric.3') {
return mocks.emptyMetricResponse;
}
return mocks.basicMetricResponse;
});
Expand Down Expand Up @@ -161,9 +163,9 @@ describe('The metric threshold alert type', () => {
await execute(Comparator.GT, [0.75]);
const { action } = mostRecentAction(instanceID);
expect(action.group).toBe('*');
expect(action.valueOf.condition0).toBe(1);
expect(action.thresholdOf.condition0).toStrictEqual([0.75]);
expect(action.metricOf.condition0).toBe('test.metric.1');
expect(action.reason).toContain('current value is 1');
expect(action.reason).toContain('threshold of 0.75');
expect(action.reason).toContain('test.metric.1');
});
test('fetches the index pattern dynamically', async () => {
await execute(Comparator.LT, [17], 'alternate');
Expand Down Expand Up @@ -271,12 +273,14 @@ describe('The metric threshold alert type', () => {
const instanceID = 'test-*';
await execute(Comparator.GT_OR_EQ, [1.0], [3.0]);
const { action } = mostRecentAction(instanceID);
expect(action.valueOf.condition0).toBe(1);
expect(action.valueOf.condition1).toBe(3.5);
expect(action.thresholdOf.condition0).toStrictEqual([1.0]);
expect(action.thresholdOf.condition1).toStrictEqual([3.0]);
expect(action.metricOf.condition0).toBe('test.metric.1');
expect(action.metricOf.condition1).toBe('test.metric.2');
const reasons = action.reason.split('\n');
expect(reasons.length).toBe(2);
expect(reasons[0]).toContain('test.metric.1');
expect(reasons[1]).toContain('test.metric.2');
expect(reasons[0]).toContain('current value is 1');
expect(reasons[1]).toContain('current value is 3.5');
expect(reasons[0]).toContain('threshold of 1');
expect(reasons[1]).toContain('threshold of 3');
});
});
describe('querying with the count aggregator', () => {
Expand Down Expand Up @@ -305,4 +309,32 @@ describe('The metric threshold alert type', () => {
expect(getState(instanceID).alertState).toBe(AlertStates.OK);
});
});
describe("querying a metric that hasn't reported data", () => {
const instanceID = 'test-*';
const execute = (alertOnNoData: boolean) =>
executor({
services,
params: {
criteria: [
{
...baseCriterion,
comparator: Comparator.GT,
threshold: 1,
metric: 'test.metric.3',
},
],
alertOnNoData,
},
});
test('sends a No Data alert when configured to do so', async () => {
await execute(true);
expect(mostRecentAction(instanceID).id).toBe(FIRED_ACTIONS.id);
expect(getState(instanceID).alertState).toBe(AlertStates.NO_DATA);
});
test('does not send a No Data alert when not configured to do so', async () => {
await execute(false);
expect(mostRecentAction(instanceID)).toBe(undefined);
expect(getState(instanceID).alertState).toBe(AlertStates.NO_DATA);
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,13 @@ import { createAfterKeyHandler } from '../../../utils/create_afterkey_handler';
import { getAllCompositeData } from '../../../utils/get_all_composite_data';
import { networkTraffic } from '../../../../common/inventory_models/shared/metrics/snapshot/network_traffic';
import { MetricExpressionParams, Comparator, Aggregators, AlertStates } from './types';
import {
buildErrorAlertReason,
buildFiredAlertReason,
buildNoDataAlertReason,
DOCUMENT_COUNT_I18N,
stateToAlertMessage,
} from './messages';
import { AlertServices, AlertExecutorOptions } from '../../../../../alerting/server';
import { getIntervalInSeconds } from '../../../utils/get_interval_in_seconds';
import { getDateHistogramOffset } from '../../snapshot/query_helpers';
Expand Down Expand Up @@ -258,24 +265,14 @@ const comparatorMap = {
[Comparator.LT_OR_EQ]: (a: number, [b]: number[]) => a <= b,
};

const mapToConditionsLookup = (
list: any[],
mapFn: (value: any, index: number, array: any[]) => unknown
) =>
list
.map(mapFn)
.reduce(
(result: Record<string, any>, value, i) => ({ ...result, [`condition${i}`]: value }),
{}
);

export const createMetricThresholdExecutor = (alertUUID: string) =>
async function({ services, params }: AlertExecutorOptions) {
const { criteria, groupBy, filterQuery, sourceId } = params as {
const { criteria, groupBy, filterQuery, sourceId, alertOnNoData } = params as {
criteria: MetricExpressionParams[];
groupBy: string | undefined;
filterQuery: string | undefined;
sourceId?: string;
alertOnNoData: boolean;
};

const alertResults = await Promise.all(
Expand All @@ -286,9 +283,11 @@ export const createMetricThresholdExecutor = (alertUUID: string) =>
const { threshold, comparator } = criterion;
const comparisonFunction = comparatorMap[comparator];
return mapValues(currentValues, value => ({
...criterion,
metric: criterion.metric ?? DOCUMENT_COUNT_I18N,
currentValue: value,
shouldFire:
value !== undefined && value !== null && comparisonFunction(value, threshold),
currentValue: value,
isNoData: value === null,
isError: value === undefined,
}));
Expand All @@ -306,23 +305,43 @@ export const createMetricThresholdExecutor = (alertUUID: string) =>
// whole alert is in a No Data/Error state
const isNoData = alertResults.some(result => result[group].isNoData);
const isError = alertResults.some(result => result[group].isError);
if (shouldAlertFire) {

const nextState = isError
? AlertStates.ERROR
: isNoData
? AlertStates.NO_DATA
: shouldAlertFire
? AlertStates.ALERT
: AlertStates.OK;

let reason;
if (nextState === AlertStates.ALERT) {
reason = alertResults.map(result => buildFiredAlertReason(result[group])).join('\n');
}
if (alertOnNoData) {
if (nextState === AlertStates.NO_DATA) {
reason = alertResults
.filter(result => result[group].isNoData)
.map(result => buildNoDataAlertReason(result[group]))
.join('\n');
} else if (nextState === AlertStates.ERROR) {
reason = alertResults
.filter(result => result[group].isError)
.map(result => buildErrorAlertReason(result[group].metric))
.join('\n');
}
}
if (reason) {
alertInstance.scheduleActions(FIRED_ACTIONS.id, {
group,
valueOf: mapToConditionsLookup(alertResults, result => result[group].currentValue),
thresholdOf: mapToConditionsLookup(criteria, criterion => criterion.threshold),
metricOf: mapToConditionsLookup(criteria, criterion => criterion.metric),
alertState: stateToAlertMessage[nextState],
reason,
});
}

// Future use: ability to fetch display current alert state
alertInstance.replaceState({
alertState: isError
? AlertStates.ERROR
: isNoData
? AlertStates.NO_DATA
: shouldAlertFire
? AlertStates.ALERT
: AlertStates.OK,
alertState: nextState,
});
}
};
Expand Down
Loading