[APM] Alert annotations

Closes elastic#60373.
dgieselaar · May 19, 2020 · 6cc089b · 6cc089b
1 parent 3350bff
commit 6cc089b
Show file tree

Hide file tree

Showing 14 changed files with 117 additions and 61 deletions.
diff --git a/x-pack/plugins/apm/common/annotations.ts b/x-pack/plugins/apm/common/annotations.ts
@@ -5,7 +5,8 @@
  */
 
 export enum AnnotationType {
-  VERSION = 'version'
+  VERSION = 'version',
+  ALERT = 'alert'
 }
 
 export interface Annotation {

diff --git a/x-pack/plugins/apm/public/components/shared/charts/CustomPlot/AnnotationsPlot.tsx b/x-pack/plugins/apm/public/components/shared/charts/CustomPlot/AnnotationsPlot.tsx
@@ -15,7 +15,7 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { Maybe } from '../../../../../typings/common';
-import { Annotation } from '../../../../../common/annotations';
+import { Annotation, AnnotationType } from '../../../../../common/annotations';
 import { PlotValues, SharedPlot } from './plotUtils';
 import { asAbsoluteDateTime } from '../../../../utils/formatters';
 
@@ -31,6 +31,11 @@ const style = {
   strokeDasharray: 'none'
 };
 
+const colorMap = {
+  [AnnotationType.VERSION]: theme.euiColorSecondary,
+  [AnnotationType.ALERT]: theme.euiColorVis3
+};
+
 export function AnnotationsPlot(props: Props) {
   const { plotValues, annotations } = props;
 
@@ -56,16 +61,16 @@ export function AnnotationsPlot(props: Props) {
               <EuiFlexGroup>
                 <EuiFlexItem grow={true}>
                   <EuiText>
-                    {i18n.translate('xpack.apm.version', {
-                      defaultMessage: 'Version'
+                    {i18n.translate('xpack.apm.annotation.message', {
+                      defaultMessage: 'Message'
                     })}
                   </EuiText>
                 </EuiFlexItem>
                 <EuiFlexItem grow={false}>{annotation.text}</EuiFlexItem>
               </EuiFlexGroup>
             }
           >
-            <EuiIcon type="dot" color={theme.euiColorSecondary} />
+            <EuiIcon type="dot" color={colorMap[annotation.type]} />
           </EuiToolTip>
         </div>
       ))}

diff --git a/x-pack/plugins/apm/public/components/shared/charts/CustomPlot/Legends.js b/x-pack/plugins/apm/public/components/shared/charts/CustomPlot/Legends.js
@@ -122,8 +122,8 @@ export default function Legends({
           }}
           text={
             <LegendContent>
-              {i18n.translate('xpack.apm.serviceVersion', {
-                defaultMessage: 'Service version'
+              {i18n.translate('xpack.apm.legends.annotations', {
+                defaultMessage: 'Annotations'
               })}
             </LegendContent>
           }

diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts
@@ -5,25 +5,27 @@
  */
 
 import { Observable } from 'rxjs';
+import { ObservabilityPluginSetup } from '../../../../observability/server';
 import { AlertingPlugin } from '../../../../alerting/server';
-import { ActionsPlugin } from '../../../../actions/server';
 import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type';
 import { registerErrorRateAlertType } from './register_error_rate_alert_type';
 import { APMConfig } from '../..';
 
 interface Params {
   alerting: AlertingPlugin['setup'];
-  actions: ActionsPlugin['setup'];
+  observability?: ObservabilityPluginSetup;
   config$: Observable<APMConfig>;
 }
 
 export function registerApmAlerts(params: Params) {
   registerTransactionDurationAlertType({
     alerting: params.alerting,
+    observability: params.observability,
     config$: params.config$
   });
   registerErrorRateAlertType({
     alerting: params.alerting,
+    observability: params.observability,
     config$: params.config$
   });
 }
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_rate_alert_type.ts
@@ -8,6 +8,7 @@ import { schema, TypeOf } from '@kbn/config-schema';
 import { Observable } from 'rxjs';
 import { take } from 'rxjs/operators';
 import { i18n } from '@kbn/i18n';
+import { ObservabilityPluginSetup } from '../../../../observability/server';
 import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
 import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types';
 import {
@@ -25,6 +26,7 @@ import { APMConfig } from '../..';
 
 interface RegisterAlertParams {
   alerting: AlertingPlugin['setup'];
+  observability?: ObservabilityPluginSetup;
   config$: Observable<APMConfig>;
 }
 
@@ -40,6 +42,7 @@ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.ErrorRate];
 
 export function registerErrorRateAlertType({
   alerting,
+  observability,
   config$
 }: RegisterAlertParams) {
   alerting.registerType({

diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts
@@ -8,6 +8,7 @@ import { schema, TypeOf } from '@kbn/config-schema';
 import { Observable } from 'rxjs';
 import { take } from 'rxjs/operators';
 import { i18n } from '@kbn/i18n';
+import { ObservabilityPluginSetup } from '../../../../observability/server';
 import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values';
 import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types';
 import { ESSearchResponse } from '../../../typings/elasticsearch';
@@ -24,6 +25,7 @@ import { APMConfig } from '../..';
 
 interface RegisterAlertParams {
   alerting: AlertingPlugin['setup'];
+  observability?: ObservabilityPluginSetup;
   config$: Observable<APMConfig>;
 }
 
@@ -44,6 +46,7 @@ const paramsSchema = schema.object({
 const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDuration];
 
 export function registerTransactionDurationAlertType({
+  observability,
   alerting,
   config$
 }: RegisterAlertParams) {
@@ -81,17 +84,25 @@ export function registerTransactionDurationAlertType({
     executor: async ({ services, params }) => {
       const config = await config$.pipe(take(1)).toPromise();
 
-      const alertParams = params as TypeOf<typeof paramsSchema>;
+      const {
+        serviceName,
+        environment,
+        threshold,
+        transactionType,
+        windowSize,
+        windowUnit,
+        aggregationType
+      } = params as TypeOf<typeof paramsSchema>;
 
       const indices = await getApmIndices({
         config,
         savedObjectsClient: services.savedObjectsClient
       });
 
       const environmentTerm =
-        alertParams.environment === ENVIRONMENT_ALL
+        environment === ENVIRONMENT_ALL
           ? []
-          : [{ term: { [SERVICE_ENVIRONMENT]: alertParams.environment } }];
+          : [{ term: { [SERVICE_ENVIRONMENT]: environment } }];
 
       const searchParams = {
         index: indices['apm_oss.transactionIndices'],
@@ -103,7 +114,7 @@ export function registerTransactionDurationAlertType({
                 {
                   range: {
                     '@timestamp': {
-                      gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`
+                      gte: `now-${windowSize}${windowUnit}`
                     }
                   }
                 },
@@ -114,12 +125,12 @@ export function registerTransactionDurationAlertType({
                 },
                 {
                   term: {
-                    [SERVICE_NAME]: alertParams.serviceName
+                    [SERVICE_NAME]: serviceName
                   }
                 },
                 {
                   term: {
-                    [TRANSACTION_TYPE]: alertParams.transactionType
+                    [TRANSACTION_TYPE]: transactionType
                   }
                 },
                 ...environmentTerm
@@ -128,7 +139,7 @@ export function registerTransactionDurationAlertType({
           },
           aggs: {
             agg:
-              alertParams.aggregationType === 'avg'
+              aggregationType === 'avg'
                 ? {
                     avg: {
                       field: TRANSACTION_DURATION
@@ -137,9 +148,7 @@ export function registerTransactionDurationAlertType({
                 : {
                     percentiles: {
                       field: TRANSACTION_DURATION,
-                      percents: [
-                        alertParams.aggregationType === '95th' ? 95 : 99
-                      ]
+                      percents: [aggregationType === '95th' ? 95 : 99]
                     }
                   }
           }
@@ -159,13 +168,39 @@ export function registerTransactionDurationAlertType({
 
       const value = 'values' in agg ? agg.values[0] : agg.value;
 
-      if (value && value > alertParams.threshold * 1000) {
+      if (value && value > threshold * 1000) {
         const alertInstance = services.alertInstanceFactory(
           AlertType.TransactionDuration
         );
+
         alertInstance.scheduleActions(alertTypeConfig.defaultActionGroupId, {
-          transactionType: alertParams.transactionType,
-          serviceName: alertParams.serviceName
+          transactionType,
+          serviceName
+        });
+
+        const annotationsClient = await observability?.getScopedAnnotationsClient(
+          services.callCluster
+        );
+
+        await annotationsClient?.create({
+          '@timestamp': new Date().toISOString(),
+          annotation: {
+            type: 'alert'
+          },
+          message: i18n.translate(
+            'xpack.apm.registerTransactionDurationAlertType.annotation',
+            {
+              defaultMessage: `Transaction duration exceeded`
+            }
+          ),
+          tags: ['apm'],
+          service: {
+            name: serviceName,
+            ...(environment !== ENVIRONMENT_ALL ? { environment } : {})
+          },
+          transaction: {
+            type: transactionType
+          }
         });
       }
 

diff --git a/x-pack/plugins/apm/server/lib/services/annotations/get_stored_annotations.ts b/x-pack/plugins/apm/server/lib/services/annotations/get_stored_annotations.ts
@@ -46,7 +46,7 @@ export async function getStoredAnnotations({
                     }
                   }
                 },
-                { term: { 'annotation.type': 'deployment' } },
+                { terms: { 'annotation.type': ['deployment', 'alert'] } },
                 { term: { tags: 'apm' } },
                 { term: { [SERVICE_NAME]: serviceName } },
                 ...(environmentFilter ? [environmentFilter] : [])
@@ -59,7 +59,10 @@ export async function getStoredAnnotations({
 
     return response.hits.hits.map(hit => {
       return {
-        type: AnnotationType.VERSION,
+        type:
+          hit._source.annotation.type === 'deployment'
+            ? AnnotationType.VERSION
+            : AnnotationType.ALERT,
         id: hit._id,
         '@timestamp': new Date(hit._source['@timestamp']).getTime(),
         text: hit._source.message

diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts
@@ -17,7 +17,6 @@ import { SecurityPluginSetup } from '../../security/public';
 import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { TaskManagerSetupContract } from '../../task_manager/server';
 import { AlertingPlugin } from '../../alerting/server';
-import { ActionsPlugin } from '../../actions/server';
 import { APMOSSPluginSetup } from '../../../../src/plugins/apm_oss/server';
 import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index';
 import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index';
@@ -57,7 +56,6 @@ export class APMPlugin implements Plugin<APMPluginSetup> {
       usageCollection?: UsageCollectionSetup;
       taskManager?: TaskManagerSetupContract;
       alerting?: AlertingPlugin['setup'];
-      actions?: ActionsPlugin['setup'];
       observability?: ObservabilityPluginSetup;
       features: FeaturesPluginSetup;
       security?: SecurityPluginSetup;
@@ -72,10 +70,10 @@ export class APMPlugin implements Plugin<APMPluginSetup> {
     core.savedObjects.registerType(apmIndices);
     core.savedObjects.registerType(apmTelemetry);
 
-    if (plugins.actions && plugins.alerting) {
+    if (plugins.alerting) {
       registerApmAlerts({
         alerting: plugins.alerting,
-        actions: plugins.actions,
+        observability: plugins.observability,
         config$: mergedConfig$
       });
     }

diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts
@@ -95,8 +95,7 @@ export const serviceAnnotationsRoute = createRoute(() => ({
     const { environment } = context.params.query;
 
     const annotationsClient = await context.plugins.observability?.getScopedAnnotationsClient(
-      context,
-      request
+      context.core.elasticsearch.dataClient.callAsCurrentUser
     );
 
     return getServiceAnnotations({
@@ -139,8 +138,7 @@ export const serviceAnnotationsCreateRoute = createRoute(() => ({
   },
   handler: async ({ request, context }) => {
     const annotationsClient = await context.plugins.observability?.getScopedAnnotationsClient(
-      context,
-      request
+      context.core.elasticsearch.dataClient.callAsCurrentUser
     );
 
     if (!annotationsClient) {

diff --git a/x-pack/plugins/observability/common/annotations.ts b/x-pack/plugins/observability/common/annotations.ts
@@ -22,6 +22,9 @@ export const createAnnotationRt = t.intersection([
       environment: t.string,
       version: t.string,
     }),
+    transaction: t.type({
+      type: t.string,
+    }),
   }),
 ]);
 

diff --git a/x-pack/plugins/observability/server/lib/annotations/bootstrap_annotations.ts b/x-pack/plugins/observability/server/lib/annotations/bootstrap_annotations.ts
@@ -3,12 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import {
-  CoreSetup,
-  PluginInitializerContext,
-  KibanaRequest,
-  RequestHandlerContext,
-} from 'kibana/server';
+import { CoreSetup, PluginInitializerContext, APICaller } from 'kibana/server';
+import { ILicense } from '../../../../licensing/public';
 import { PromiseReturnType } from '../../../typings/common';
 import { createAnnotationsClient } from './create_annotations_client';
 import { registerAnnotationAPIs } from './register_annotation_apis';
@@ -19,7 +15,7 @@ interface Params {
   context: PluginInitializerContext;
 }
 
-export type ScopedAnnotationsClientFactory = PromiseReturnType<
+type ScopedAnnotationsClientFactory = PromiseReturnType<
   typeof bootstrapAnnotations
 >['getScopedAnnotationsClient'];
 
@@ -36,12 +32,18 @@ export async function bootstrapAnnotations({ index, core, context }: Params) {
   });
 
   return {
-    getScopedAnnotationsClient: (requestContext: RequestHandlerContext, request: KibanaRequest) => {
+    getScopedAnnotationsClient: ({
+      apiCaller,
+      license,
+    }: {
+      apiCaller: APICaller;
+      license?: ILicense;
+    }) => {
       return createAnnotationsClient({
         index,
-        apiCaller: core.elasticsearch.dataClient.asScoped(request).callAsCurrentUser,
+        apiCaller,
         logger,
-        license: requestContext.licensing?.license,
+        license,
       });
     },
   };