Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Metrics Alerts] Create Metric Threshold Alert Type and Executor #57606

Merged
merged 24 commits into from
Feb 27, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
f33df05
[Infra] Add basic backend for metric threshold alerts
Zacqary Oct 2, 2019
96ef0bd
Merge remote-tracking branch 'upstream/master' into 46511-alerting-poc
Zacqary Oct 2, 2019
4e30bb6
Define separate fired/recovered action groups
Zacqary Oct 3, 2019
cb27ef3
Allow alerting on arbitrary search fields besides host.name
Zacqary Oct 3, 2019
d64d0df
Add list and delete endpoioints
Zacqary Oct 7, 2019
5eb056a
Add groupBy alerts
Zacqary Oct 8, 2019
0de90d8
Merge in Alerting POC
Zacqary Feb 3, 2020
1a55e7f
Remove extraneous routes and SavedObject logic
Zacqary Feb 13, 2020
78bc666
Remove additional SavedObject code
Zacqary Feb 13, 2020
0fe4c0c
Remove renotify logic from executor
Zacqary Feb 13, 2020
30babc2
Merge branch 'master' into 56427-metric-alert-type
elasticmachine Feb 13, 2020
65de9eb
Fix action group type
Zacqary Feb 14, 2020
6384f23
Merge branch '56427-metric-alert-type' of github.com:Zacqary/kibana i…
Zacqary Feb 14, 2020
24d6fa6
Fix scheduledActions typecheck
Zacqary Feb 14, 2020
44082e1
Fix i18n
Zacqary Feb 14, 2020
3343719
Merge remote-tracking branch 'upstream/master' into 56427-metric-aler…
Zacqary Feb 18, 2020
d4693d5
Migrate alerting to new platform
Zacqary Feb 18, 2020
58b6489
Add alerting to infra dependencies
Zacqary Feb 19, 2020
4264930
Merge branch 'master' into 56427-metric-alert-type
elasticmachine Feb 20, 2020
09ac117
Add comment about future use
Zacqary Feb 21, 2020
ae0bd9d
Merge branch '56427-metric-alert-type' of github.com:Zacqary/kibana i…
Zacqary Feb 21, 2020
a88d0cf
Merge branch 'master' into 56427-metric-alert-type
elasticmachine Feb 26, 2020
54441b5
Adjust alert params tm names to sync with UI; default to Entire Infra…
Zacqary Feb 26, 2020
7990079
Add support for between comparator
Zacqary Feb 26, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion x-pack/plugins/infra/kibana.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,17 @@
"id": "infra",
"version": "8.0.0",
"kibanaVersion": "kibana",
"requiredPlugins": ["features", "apm", "usageCollection", "spaces", "home", "data", "data_enhanced", "metrics"],
"requiredPlugins": [
"features",
"apm",
"usageCollection",
"spaces",
"home",
"data",
"data_enhanced",
"metrics",
"alerting"
],
"server": true,
"ui": true,
"configPath": ["xpack", "infra"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { SpacesPluginSetup } from '../../../../../../plugins/spaces/server';
import { VisTypeTimeseriesSetup } from '../../../../../../../src/plugins/vis_type_timeseries/server';
import { APMPluginContract } from '../../../../../../plugins/apm/server';
import { HomeServerPluginSetup } from '../../../../../../../src/plugins/home/server';
import { PluginSetupContract as AlertingPluginContract } from '../../../../../../plugins/alerting/server';

// NP_TODO: Compose real types from plugins we depend on, no "any"
export interface InfraServerPluginDeps {
Expand All @@ -25,6 +26,7 @@ export interface InfraServerPluginDeps {
};
features: FeaturesPluginSetup;
apm: APMPluginContract;
alerting: AlertingPluginContract;
}

export interface CallWithRequestParams extends GenericParams {
Expand Down
7 changes: 7 additions & 0 deletions x-pack/plugins/infra/server/lib/alerting/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

export { registerAlertTypes } from './register_alert_types';
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import uuid from 'uuid';
import { i18n } from '@kbn/i18n';
import { schema } from '@kbn/config-schema';
import {
MetricThresholdAlertTypeParams,
Comparator,
AlertStates,
METRIC_THRESHOLD_ALERT_TYPE_ID,
} from './types';
import { AlertServices, PluginSetupContract } from '../../../../../alerting/server';

const FIRED_ACTIONS = {
id: 'metrics.threshold.fired',
name: i18n.translate('xpack.infra.metrics.alerting.threshold.fired', {
defaultMessage: 'Fired',
}),
};

async function getMetric(
{ callCluster }: AlertServices,
{ metric, aggType, timeUnit, timeSize, indexPattern }: MetricThresholdAlertTypeParams
) {
const interval = `${timeSize}${timeUnit}`;
const searchBody = {
query: {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: `now-${interval}`,
},
},
exists: {
field: metric,
},
},
],
},
},
size: 0,
aggs: {
aggregatedIntervals: {
date_histogram: {
field: '@timestamp',
fixed_interval: interval,
},
aggregations: {
aggregatedValue: {
[aggType]: {
field: metric,
},
},
},
},
},
};

const result = await callCluster('search', {
body: searchBody,
index: indexPattern,
});

const { buckets } = result.aggregations.aggregatedIntervals;
const { value } = buckets[buckets.length - 1].aggregatedValue;
return value;
}

const comparatorMap = {
[Comparator.BETWEEN]: (value: number, [a, b]: number[]) =>
value >= Math.min(a, b) && value <= Math.max(a, b),
// `threshold` is always an array of numbers in case the BETWEEN comparator is
// used; all other compartors will just destructure the first value in the array
[Comparator.GT]: (a: number, [b]: number[]) => a > b,
[Comparator.LT]: (a: number, [b]: number[]) => a < b,
[Comparator.GT_OR_EQ]: (a: number, [b]: number[]) => a >= b,
[Comparator.LT_OR_EQ]: (a: number, [b]: number[]) => a <= b,
};

export async function registerMetricThresholdAlertType(alertingPlugin: PluginSetupContract) {
if (!alertingPlugin) {
throw new Error(
'Cannot register metric threshold alert type. Both the actions and alerting plugins need to be enabled.'
);
}
const alertUUID = uuid.v4();

alertingPlugin.registerType({
id: METRIC_THRESHOLD_ALERT_TYPE_ID,
name: 'Metric Alert - Threshold',
validate: {
params: schema.object({
threshold: schema.arrayOf(schema.number()),
comparator: schema.string(),
aggType: schema.string(),
metric: schema.string(),
timeUnit: schema.string(),
timeSize: schema.number(),
indexPattern: schema.string(),
}),
},
defaultActionGroupId: FIRED_ACTIONS.id,
actionGroups: [FIRED_ACTIONS],
async executor({ services, params }) {
const { threshold, comparator } = params as MetricThresholdAlertTypeParams;
const alertInstance = services.alertInstanceFactory(alertUUID);
const currentValue = await getMetric(services, params as MetricThresholdAlertTypeParams);
if (typeof currentValue === 'undefined')
throw new Error('Could not get current value of metric');

const comparisonFunction = comparatorMap[comparator];

const isValueInAlertState = comparisonFunction(currentValue, threshold);

if (isValueInAlertState) {
alertInstance.scheduleActions(FIRED_ACTIONS.id, {
value: currentValue,
});
}

// Future use: ability to fetch display current alert state
alertInstance.replaceState({
alertState: isValueInAlertState ? AlertStates.ALERT : AlertStates.OK,
});
},
});
}
34 changes: 34 additions & 0 deletions x-pack/plugins/infra/server/lib/alerting/metric_threshold/types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { MetricsExplorerAggregation } from '../../../../common/http_api/metrics_explorer';

export const METRIC_THRESHOLD_ALERT_TYPE_ID = 'metrics.alert.threshold';

export enum Comparator {
GT = '>',
LT = '<',
GT_OR_EQ = '>=',
LT_OR_EQ = '<=',
BETWEEN = 'between',
}

export enum AlertStates {
OK,
ALERT,
}

export type TimeUnit = 's' | 'm' | 'h' | 'd';

export interface MetricThresholdAlertTypeParams {
aggType: MetricsExplorerAggregation;
metric: string;
timeSize: number;
timeUnit: TimeUnit;
indexPattern: string;
threshold: number[];
comparator: Comparator;
}
20 changes: 20 additions & 0 deletions x-pack/plugins/infra/server/lib/alerting/register_alert_types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

import { PluginSetupContract } from '../../../../alerting/server';
import { registerMetricThresholdAlertType } from './metric_threshold/register_metric_threshold_alert_type';

const registerAlertTypes = (alertingPlugin: PluginSetupContract) => {
if (alertingPlugin) {
const registerFns = [registerMetricThresholdAlertType];

registerFns.forEach(fn => {
fn(alertingPlugin);
});
}
};

export { registerAlertTypes };
2 changes: 2 additions & 0 deletions x-pack/plugins/infra/server/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import { InfraServerPluginDeps } from './lib/adapters/framework';
import { METRICS_FEATURE, LOGS_FEATURE } from './features';
import { UsageCollector } from './usage/usage_collector';
import { InfraStaticSourceConfiguration } from './lib/sources/types';
import { registerAlertTypes } from './lib/alerting';

export const config = {
schema: schema.object({
Expand Down Expand Up @@ -146,6 +147,7 @@ export class InfraServerPlugin {
]);

initInfraServer(this.libs);
registerAlertTypes(plugins.alerting);

// Telemetry
UsageCollector.registerUsageCollector(plugins.usageCollection);
Expand Down