From c0d6057b55c8be9e7f48ae2255beb8ff3c7ff19a Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 10 Nov 2020 16:05:11 +0100 Subject: [PATCH 1/4] [APM] Errors table for service overview Closes #81725. --- .../runtime_types/to_number_rt/index.ts | 16 ++ .../ServiceList/ServiceListMetric.tsx | 33 +-- .../components/app/service_overview/index.tsx | 34 +-- .../fetch_wrapper.tsx | 30 ++ .../service_overview_errors_table/index.tsx | 264 ++++++++++++++++++ .../service_overview/table_link_flex_item.tsx | 14 + .../{SparkPlot => spark_plot}/index.tsx | 0 .../spark_plot_with_value_label/index.tsx | 57 ++++ .../apm/server/lib/errors/get_error_groups.ts | 4 +- .../apm/server/lib/helpers/get_error_name.ts | 11 + .../get_service_error_groups/index.ts | 132 +++++++++ .../apm/server/routes/create_apm_api.ts | 2 + x-pack/plugins/apm/server/routes/services.ts | 44 +++ .../apm_api_integration/basic/tests/index.ts | 4 + .../tests/service_overview/error_groups.ts | 216 ++++++++++++++ 15 files changed, 803 insertions(+), 58 deletions(-) create mode 100644 x-pack/plugins/apm/common/runtime_types/to_number_rt/index.ts create mode 100644 x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/fetch_wrapper.tsx create mode 100644 x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx create mode 100644 x-pack/plugins/apm/public/components/app/service_overview/table_link_flex_item.tsx rename x-pack/plugins/apm/public/components/shared/charts/{SparkPlot => spark_plot}/index.tsx (100%) create mode 100644 x-pack/plugins/apm/public/components/shared/charts/spark_plot/spark_plot_with_value_label/index.tsx create mode 100644 x-pack/plugins/apm/server/lib/helpers/get_error_name.ts create mode 100644 x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts create mode 100644 x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts diff --git a/x-pack/plugins/apm/common/runtime_types/to_number_rt/index.ts b/x-pack/plugins/apm/common/runtime_types/to_number_rt/index.ts new file mode 100644 index 0000000000000..0fe8181c11405 --- /dev/null +++ b/x-pack/plugins/apm/common/runtime_types/to_number_rt/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import * as t from 'io-ts'; + +export const toNumberRt = new t.Type( + 'ToNumber', + t.any.is, + (input, context) => { + const number = Number(input); + return !isNaN(number) ? t.success(number) : t.failure(input, context); + }, + t.identity +); diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/ServiceListMetric.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/ServiceListMetric.tsx index c94c94d4a0b72..716fed7775f7b 100644 --- a/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/ServiceListMetric.tsx +++ b/x-pack/plugins/apm/public/components/app/service_inventory/ServiceList/ServiceListMetric.tsx @@ -3,15 +3,9 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { EuiFlexItem } from '@elastic/eui'; -import { EuiFlexGroup } from '@elastic/eui'; - import React from 'react'; -import { useTheme } from '../../../../hooks/useTheme'; import { useUrlParams } from '../../../../hooks/useUrlParams'; -import { getEmptySeries } from '../../../shared/charts/CustomPlot/getEmptySeries'; -import { SparkPlot } from '../../../shared/charts/SparkPlot'; +import { SparkPlotWithValueLabel } from '../../../shared/charts/spark_plot/spark_plot_with_value_label'; export function ServiceListMetric({ color, @@ -22,28 +16,17 @@ export function ServiceListMetric({ series?: Array<{ x: number; y: number | null }>; valueLabel: React.ReactNode; }) { - const theme = useTheme(); - const { urlParams: { start, end }, } = useUrlParams(); - const colorValue = theme.eui[color]; - return ( - - - - - - {valueLabel} - - + ); } diff --git a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx index 016ee3daf6b51..ee77157fe4eb3 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/index.tsx @@ -12,9 +12,10 @@ import { useTrackPageview } from '../../../../../observability/public'; import { isRumAgentName } from '../../../../common/agent_name'; import { ChartsSyncContextProvider } from '../../../context/charts_sync_context'; import { TransactionErrorRateChart } from '../../shared/charts/transaction_error_rate_chart'; -import { ErrorOverviewLink } from '../../shared/Links/apm/ErrorOverviewLink'; import { ServiceMapLink } from '../../shared/Links/apm/ServiceMapLink'; import { TransactionOverviewLink } from '../../shared/Links/apm/TransactionOverviewLink'; +import { ServiceOverviewErrorsTable } from './service_overview_errors_table'; +import { TableLinkFlexItem } from './table_link_flex_item'; const rowHeight = 310; const latencyChartRowHeight = 230; @@ -27,12 +28,6 @@ const LatencyChartRow = styled(EuiFlexItem)` height: ${latencyChartRowHeight}px; `; -const TableLinkFlexItem = styled(EuiFlexItem)` - & > a { - text-align: right; - } -`; - interface ServiceOverviewProps { agentName?: string; serviceName: string; @@ -130,30 +125,7 @@ export function ServiceOverview({ )} - - - -

- {i18n.translate( - 'xpack.apm.serviceOverview.errorsTableTitle', - { - defaultMessage: 'Errors', - } - )} -

- - - - - {i18n.translate( - 'xpack.apm.serviceOverview.errorsTableLinkText', - { - defaultMessage: 'View errors', - } - )} - - - + diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/fetch_wrapper.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/fetch_wrapper.tsx new file mode 100644 index 0000000000000..4c8d368811a0c --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/fetch_wrapper.tsx @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { FETCH_STATUS } from '../../../../hooks/useFetcher'; +import { ErrorStatePrompt } from '../../../shared/ErrorStatePrompt'; +import { LoadingStatePrompt } from '../../../shared/LoadingStatePrompt'; + +export function FetchWrapper({ + hasData, + status, + children, +}: { + hasData: boolean; + status: FETCH_STATUS; + children: React.ReactNode; +}) { + if (status === FETCH_STATUS.FAILURE) { + return ; + } + + if (!hasData && status !== FETCH_STATUS.SUCCESS) { + return ; + } + + return <>{children}; +} diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx new file mode 100644 index 0000000000000..9cf662a20c326 --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { useState } from 'react'; +import { EuiTitle } from '@elastic/eui'; +import { EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { EuiBasicTable } from '@elastic/eui'; +import { EuiBasicTableColumn } from '@elastic/eui'; +import styled from 'styled-components'; +import { EuiToolTip } from '@elastic/eui'; +import { FETCH_STATUS, useFetcher } from '../../../../hooks/useFetcher'; +import { useUrlParams } from '../../../../hooks/useUrlParams'; +import { ErrorOverviewLink } from '../../../shared/Links/apm/ErrorOverviewLink'; +import { TableLinkFlexItem } from '../table_link_flex_item'; +import { SparkPlotWithValueLabel } from '../../../shared/charts/spark_plot/spark_plot_with_value_label'; +import { callApmApi } from '../../../../services/rest/createCallApmApi'; +import { TimestampTooltip } from '../../../shared/TimestampTooltip'; +import { ErrorDetailLink } from '../../../shared/Links/apm/ErrorDetailLink'; +import { truncate } from '../../../../style/variables'; +import { FetchWrapper } from './fetch_wrapper'; + +interface Props { + serviceName: string; +} + +interface ErrorGroupItem { + name: string; + last_seen: number; + group_id: string; + occurrences: { + value: number | null; + timeseries: Array<{ x: number; y: number }>; + }; +} + +type SortDirection = 'asc' | 'desc'; +type SortField = 'name' | 'last_seen' | 'occurrences'; + +const PAGE_SIZE = 5; +const DEFAULT_SORT = { + direction: 'desc' as const, + field: 'occurrences' as const, +}; + +const ErrorDetailLinkWrapper = styled.div` + width: 100%; + .euiToolTipAnchor { + width: 100% !important; + } +`; + +const StyledErrorDetailLink = styled(ErrorDetailLink)` + display: block; + ${truncate('100%')} +`; + +export function ServiceOverviewErrorsTable({ serviceName }: Props) { + const { + urlParams: { start, end }, + uiFilters, + } = useUrlParams(); + + const [tableOptions, setTableOptions] = useState<{ + pageIndex: number; + sort: { + direction: SortDirection; + field: SortField; + }; + }>({ + pageIndex: 0, + sort: DEFAULT_SORT, + }); + + const columns: Array> = [ + { + field: 'name', + name: i18n.translate('xpack.apm.serviceOverview.errorsTableColumnName', { + defaultMessage: 'Name', + }), + width: '70%', + render: (_, { name, group_id: errorGroupId }) => { + return ( + + + + {name} + + + + ); + }, + }, + { + field: 'last_seen', + name: i18n.translate( + 'xpack.apm.serviceOverview.errorsTableColumnLastSeen', + { + defaultMessage: 'Last seen', + } + ), + render: (_, { last_seen: lastSeen }) => { + return ; + }, + }, + { + field: 'occurrences', + name: i18n.translate( + 'xpack.apm.serviceOverview.errorsTableColumnOccurrences', + { + defaultMessage: 'Occurrences', + } + ), + render: (_, { occurrences }) => { + return ( + + ); + }, + }, + ]; + + const { + data = { + totalItemCount: 0, + items: [], + tableOptions: { + pageIndex: 0, + sort: DEFAULT_SORT, + }, + }, + status, + } = useFetcher(() => { + if (!start || !end) { + return; + } + + return callApmApi({ + pathname: '/api/apm/services/{serviceName}/error_groups', + params: { + path: { serviceName }, + query: { + start, + end, + uiFilters: JSON.stringify(uiFilters), + size: PAGE_SIZE, + numBuckets: 20, + pageIndex: tableOptions.pageIndex, + sortField: tableOptions.sort.field, + sortDirection: tableOptions.sort.direction, + }, + }, + }).then((response) => { + return { + items: response.error_groups, + totalItemCount: response.total_error_groups, + tableOptions: { + pageIndex: tableOptions.pageIndex, + sort: { + field: tableOptions.sort.field, + direction: tableOptions.sort.direction, + }, + }, + }; + }); + }, [ + start, + end, + serviceName, + uiFilters, + tableOptions.pageIndex, + tableOptions.sort.field, + tableOptions.sort.direction, + ]); + + const { + items, + totalItemCount, + tableOptions: { pageIndex, sort }, + } = data; + + return ( + + + + + +

+ {i18n.translate('xpack.apm.serviceOverview.errorsTableTitle', { + defaultMessage: 'Errors', + })} +

+ + + + + {i18n.translate('xpack.apm.serviceOverview.errorsTableLinkText', { + defaultMessage: 'View errors', + })} + + + + + + + { + setTableOptions({ + pageIndex: newTableOptions.page?.index ?? 0, + sort: newTableOptions.sort + ? { + field: newTableOptions.sort.field as SortField, + direction: newTableOptions.sort.direction, + } + : DEFAULT_SORT, + }); + }} + sorting={{ + enableAllColumns: true, + sort: { + direction: sort.direction, + field: sort.field, + }, + }} + /> + + + + ); +} diff --git a/x-pack/plugins/apm/public/components/app/service_overview/table_link_flex_item.tsx b/x-pack/plugins/apm/public/components/app/service_overview/table_link_flex_item.tsx new file mode 100644 index 0000000000000..35df003af380d --- /dev/null +++ b/x-pack/plugins/apm/public/components/app/service_overview/table_link_flex_item.tsx @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EuiFlexItem } from '@elastic/eui'; +import styled from 'styled-components'; + +export const TableLinkFlexItem = styled(EuiFlexItem)` + & > a { + text-align: right; + } +`; diff --git a/x-pack/plugins/apm/public/components/shared/charts/SparkPlot/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx similarity index 100% rename from x-pack/plugins/apm/public/components/shared/charts/SparkPlot/index.tsx rename to x-pack/plugins/apm/public/components/shared/charts/spark_plot/index.tsx diff --git a/x-pack/plugins/apm/public/components/shared/charts/spark_plot/spark_plot_with_value_label/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/spark_plot/spark_plot_with_value_label/index.tsx new file mode 100644 index 0000000000000..e2bb42fddb33b --- /dev/null +++ b/x-pack/plugins/apm/public/components/shared/charts/spark_plot/spark_plot_with_value_label/index.tsx @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { EuiFlexItem } from '@elastic/eui'; +import { EuiFlexGroup } from '@elastic/eui'; + +import React from 'react'; +import { useTheme } from '../../../../../hooks/useTheme'; +import { getEmptySeries } from '../../CustomPlot/getEmptySeries'; +import { SparkPlot } from '../'; + +type Color = + | 'euiColorVis0' + | 'euiColorVis1' + | 'euiColorVis2' + | 'euiColorVis3' + | 'euiColorVis4' + | 'euiColorVis5' + | 'euiColorVis6' + | 'euiColorVis7' + | 'euiColorVis8' + | 'euiColorVis9'; + +export function SparkPlotWithValueLabel({ + start, + end, + color, + series, + valueLabel, +}: { + start: number; + end: number; + color: Color; + series?: Array<{ x: number; y: number | null }>; + valueLabel: React.ReactNode; +}) { + const theme = useTheme(); + + const colorValue = theme.eui[color]; + + return ( + + + + + + {valueLabel} + + + ); +} diff --git a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts index 006d2fae3d4fb..4145ac13b2bee 100644 --- a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts +++ b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts @@ -17,6 +17,7 @@ import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getErrorGroupsProjection } from '../../projections/errors'; import { mergeProjection } from '../../projections/util/merge_projection'; import { SortOptions } from '../../../typings/elasticsearch/aggregations'; +import { getErrorName } from '../helpers/get_error_name'; export type ErrorGroupListAPIResponse = PromiseReturnType< typeof getErrorGroups @@ -93,8 +94,7 @@ export async function getErrorGroups({ // this is an exception rather than the rule so the ES type does not account for this. const hits = (resp.aggregations?.error_groups.buckets || []).map((bucket) => { const source = bucket.sample.hits.hits[0]._source; - const message = - source.error.log?.message || source.error.exception?.[0]?.message; + const message = getErrorName(source); return { message, diff --git a/x-pack/plugins/apm/server/lib/helpers/get_error_name.ts b/x-pack/plugins/apm/server/lib/helpers/get_error_name.ts new file mode 100644 index 0000000000000..dbc69592a4f8e --- /dev/null +++ b/x-pack/plugins/apm/server/lib/helpers/get_error_name.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { APMError } from '../../../typings/es_schemas/ui/apm_error'; + +export function getErrorName({ error }: APMError) { + return error.log?.message || error.exception?.[0]?.message; +} diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts new file mode 100644 index 0000000000000..98248e3fb2728 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ValuesType } from 'utility-types'; +import { orderBy } from 'lodash'; +import { NOT_AVAILABLE_LABEL } from '../../../../common/i18n'; +import { PromiseReturnType } from '../../../../typings/common'; +import { rangeFilter } from '../../../../common/utils/range_filter'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { + ERROR_EXC_MESSAGE, + ERROR_GROUP_ID, + ERROR_LOG_MESSAGE, + SERVICE_NAME, +} from '../../../../common/elasticsearch_fieldnames'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { getBucketSize } from '../../helpers/get_bucket_size'; +import { getErrorName } from '../../helpers/get_error_name'; + +export type ServiceErrorGroupItem = ValuesType< + PromiseReturnType +>; + +export async function getServiceErrorGroups({ + serviceName, + setup, + size, + numBuckets, + pageIndex, + sortDirection, + sortField, +}: { + serviceName: string; + setup: Setup & SetupTimeRange; + size: number; + pageIndex: number; + numBuckets: number; + sortDirection: 'asc' | 'desc'; + sortField: 'name' | 'last_seen' | 'occurrences'; +}) { + const { apmEventClient, start, end, esFilter } = setup; + + const { intervalString } = getBucketSize(start, end, numBuckets); + + const response = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.error], + }, + body: { + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { range: rangeFilter(start, end) }, + ...esFilter, + ], + }, + }, + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size: 500, + order: { + _count: 'desc', + }, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, + }, + }, + }, + sample: { + top_hits: { + size: 1, + _source: [ERROR_LOG_MESSAGE, ERROR_EXC_MESSAGE, '@timestamp'], + sort: { + '@timestamp': 'desc', + }, + }, + }, + }, + }, + }, + }, + }); + + const errorGroups = + response.aggregations?.error_groups.buckets.map((bucket) => ({ + group_id: bucket.key as string, + name: + getErrorName(bucket.sample.hits.hits[0]._source) ?? NOT_AVAILABLE_LABEL, + last_seen: new Date( + bucket.sample.hits.hits[0]?._source['@timestamp'] + ).getTime(), + occurrences: { + value: bucket.doc_count, + timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key as number, + y: dateBucket.doc_count, + })), + }, + })) ?? []; + + const sortedErrorGroups = orderBy( + errorGroups, + (group) => { + if (sortField === 'occurrences') { + return group.occurrences.value; + } + return group[sortField]; + }, + [sortDirection] + ); + + const sliceFrom = pageIndex * size; + + return { + total_error_groups: sortedErrorGroups.length, + error_groups: sortedErrorGroups.slice(sliceFrom, sliceFrom + size), + }; +} diff --git a/x-pack/plugins/apm/server/routes/create_apm_api.ts b/x-pack/plugins/apm/server/routes/create_apm_api.ts index 2fbe404a70d82..34551c35ee234 100644 --- a/x-pack/plugins/apm/server/routes/create_apm_api.ts +++ b/x-pack/plugins/apm/server/routes/create_apm_api.ts @@ -21,6 +21,7 @@ import { serviceNodeMetadataRoute, serviceAnnotationsRoute, serviceAnnotationsCreateRoute, + serviceErrorGroupsRoute, } from './services'; import { agentConfigurationRoute, @@ -115,6 +116,7 @@ const createApmApi = () => { .add(serviceNodeMetadataRoute) .add(serviceAnnotationsRoute) .add(serviceAnnotationsCreateRoute) + .add(serviceErrorGroupsRoute) // Agent configuration .add(getSingleAgentConfigurationRoute) diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 590b6c49d71bf..ada1674d4555d 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -17,6 +17,8 @@ import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceAnnotations } from '../lib/services/annotations'; import { dateAsStringRt } from '../../common/runtime_types/date_as_string_rt'; import { getSearchAggregatedTransactions } from '../lib/helpers/aggregated_transactions'; +import { getServiceErrorGroups } from '../lib/services/get_service_error_groups'; +import { toNumberRt } from '../../common/runtime_types/to_number_rt'; export const servicesRoute = createRoute(() => ({ path: '/api/apm/services', @@ -195,3 +197,45 @@ export const serviceAnnotationsCreateRoute = createRoute(() => ({ }); }, })); + +export const serviceErrorGroupsRoute = createRoute(() => ({ + path: '/api/apm/services/{serviceName}/error_groups', + params: { + path: t.type({ + serviceName: t.string, + }), + query: t.intersection([ + rangeRt, + uiFiltersRt, + t.type({ + size: toNumberRt, + numBuckets: toNumberRt, + pageIndex: toNumberRt, + sortDirection: t.union([t.literal('asc'), t.literal('desc')]), + sortField: t.union([ + t.literal('last_seen'), + t.literal('occurrences'), + t.literal('name'), + ]), + }), + ]), + }, + handler: async ({ context, request }) => { + const setup = await setupRequest(context, request); + + const { + path: { serviceName }, + query: { size, numBuckets, pageIndex, sortDirection, sortField }, + } = context.params; + + return getServiceErrorGroups({ + serviceName, + setup, + size, + numBuckets, + pageIndex, + sortDirection, + sortField, + }); + }, +})); diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index df3e60d79aca5..39dd721c7067e 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -25,6 +25,10 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont loadTestFile(require.resolve('./services/transaction_types')); }); + describe('Service overview', function () { + loadTestFile(require.resolve('./service_overview/error_groups')); + }); + describe('Settings', function () { loadTestFile(require.resolve('./settings/custom_link')); loadTestFile(require.resolve('./settings/agent_configuration')); diff --git a/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts new file mode 100644 index 0000000000000..85a278b07cf7d --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts @@ -0,0 +1,216 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import qs from 'querystring'; +import { pick, uniqBy } from 'lodash'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +import archives from '../../../common/archives_metadata'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + const archiveName = 'apm_8.0.0'; + const { start, end } = archives[archiveName]; + + describe('Service overview error groups', () => { + describe('when data is not loaded', () => { + it('handles the empty state', async () => { + const response = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size: 5, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + expect(response.status).to.be(200); + expect(response.body).to.eql({ total_error_groups: 0, error_groups: [] }); + }); + }); + + describe('when data is loaded', () => { + before(() => esArchiver.load(archiveName)); + after(() => esArchiver.unload(archiveName)); + + it('returns the correct data', async () => { + const response = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size: 5, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + expect(response.status).to.be(200); + + expectSnapshot(response.body.total_error_groups).toMatchInline(`5`); + + expectSnapshot(response.body.error_groups.map((group: any) => group.name)).toMatchInline(` + Array [ + "Could not write JSON: Null return value from advice does not match primitive return type for: public abstract double co.elastic.apm.opbeans.repositories.Numbers.getCost(); nested exception is com.fasterxml.jackson.databind.JsonMappingException: Null return value from advice does not match primitive return type for: public abstract double co.elastic.apm.opbeans.repositories.Numbers.getCost() (through reference chain: co.elastic.apm.opbeans.repositories.Stats[\\"numbers\\"]->com.sun.proxy.$Proxy133[\\"cost\\"])", + "java.io.IOException: Connection reset by peer", + "Connection reset by peer", + "Could not write JSON: Unable to find co.elastic.apm.opbeans.model.Customer with id 6617; nested exception is com.fasterxml.jackson.databind.JsonMappingException: Unable to find co.elastic.apm.opbeans.model.Customer with id 6617 (through reference chain: co.elastic.apm.opbeans.model.Customer_$$_jvst369_3[\\"email\\"])", + "Request method 'POST' not supported", + ] + `); + + expectSnapshot(response.body.error_groups.map((group: any) => group.occurrences.value)) + .toMatchInline(` + Array [ + 8, + 2, + 1, + 1, + 1, + ] + `); + + const firstItem = response.body.error_groups[0]; + + expectSnapshot(pick(firstItem, 'group_id', 'last_seen', 'name', 'occurrences.value')) + .toMatchInline(` + Object { + "group_id": "051f95eabf120ebe2f8b0399fe3e54c5", + "last_seen": 1601391561523, + "name": "Could not write JSON: Null return value from advice does not match primitive return type for: public abstract double co.elastic.apm.opbeans.repositories.Numbers.getCost(); nested exception is com.fasterxml.jackson.databind.JsonMappingException: Null return value from advice does not match primitive return type for: public abstract double co.elastic.apm.opbeans.repositories.Numbers.getCost() (through reference chain: co.elastic.apm.opbeans.repositories.Stats[\\"numbers\\"]->com.sun.proxy.$Proxy133[\\"cost\\"])", + "occurrences": Object { + "value": 8, + }, + } + `); + + expectSnapshot( + firstItem.occurrences.timeseries.filter(({ y }: any) => y > 0).length + ).toMatchInline(`7`); + }); + + it('sorts items in the correct order', async () => { + const descendingResponse = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size: 5, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + expect(descendingResponse.status).to.be(200); + + const descendingOccurrences = descendingResponse.body.error_groups.map( + (item: any) => item.occurrences.value + ); + + expect(descendingOccurrences).to.eql(descendingOccurrences.concat().sort().reverse()); + + const ascendingResponse = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size: 5, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + const ascendingOccurrences = ascendingResponse.body.error_groups.map( + (item: any) => item.occurrences.value + ); + + expect(ascendingOccurrences).to.eql(ascendingOccurrences.concat().sort().reverse()); + }); + + it('sorts items by the correct field', async () => { + const response = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size: 5, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'last_seen', + })}` + ); + + expect(response.status).to.be(200); + + const dates = response.body.error_groups.map((group: any) => group.last_seen); + + expect(dates).to.eql(dates.concat().sort().reverse()); + }); + + it('paginates through the items', async () => { + const size = 1; + + const firstPage = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size, + numBuckets: 20, + pageIndex: 0, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + expect(firstPage.status).to.eql(200); + + const totalItems = firstPage.body.total_error_groups; + + const pages = Math.floor(totalItems / size); + + const items = await new Array(pages) + .fill(undefined) + .reduce(async (prevItemsPromise, _, pageIndex) => { + const prevItems = await prevItemsPromise; + + const thisPage = await supertest.get( + `/api/apm/services/opbeans-java/error_groups?${qs.stringify({ + start, + end, + uiFilters: '{}', + size, + numBuckets: 20, + pageIndex, + sortDirection: 'desc', + sortField: 'occurrences', + })}` + ); + + return prevItems.concat(thisPage.body.error_groups); + }, Promise.resolve([])); + + expect(items.length).to.eql(totalItems); + + expect(uniqBy(items, 'group_id').length).to.eql(totalItems); + }); + }); + }); +} From f9b8298c77cea6d67631470f32b9c9f73ff47b99 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 12 Nov 2020 11:57:54 +0100 Subject: [PATCH 2/4] Only fetch timeseries data for error groups on the current page --- .../service_overview_errors_table/index.tsx | 11 +-- .../get_service_error_groups/index.ts | 83 ++++++++++++++----- .../tests/service_overview/error_groups.ts | 6 +- .../typings/elasticsearch/aggregations.d.ts | 2 + 4 files changed, 77 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx index 9cf662a20c326..b65286800a769 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx @@ -12,6 +12,7 @@ import { EuiBasicTable } from '@elastic/eui'; import { EuiBasicTableColumn } from '@elastic/eui'; import styled from 'styled-components'; import { EuiToolTip } from '@elastic/eui'; +import { asInteger } from '../../../../../common/utils/formatters'; import { FETCH_STATUS, useFetcher } from '../../../../hooks/useFetcher'; import { useUrlParams } from '../../../../hooks/useUrlParams'; import { ErrorOverviewLink } from '../../../shared/Links/apm/ErrorOverviewLink'; @@ -32,8 +33,8 @@ interface ErrorGroupItem { last_seen: number; group_id: string; occurrences: { - value: number | null; - timeseries: Array<{ x: number; y: number }>; + value: number; + timeseries: Array<{ x: number; y: number }> | null; }; } @@ -81,7 +82,7 @@ export function ServiceOverviewErrorsTable({ serviceName }: Props) { name: i18n.translate('xpack.apm.serviceOverview.errorsTableColumnName', { defaultMessage: 'Name', }), - width: '70%', + width: '60%', render: (_, { name, group_id: errorGroupId }) => { return ( @@ -121,13 +122,13 @@ export function ServiceOverviewErrorsTable({ serviceName }: Props) { return ( ({ - x: dateBucket.key as number, - y: dateBucket.doc_count, - })), }, })) ?? []; - const sortedErrorGroups = orderBy( + // Sort error groups first, and only get timeseries for data in view. + // This is to limit the possibility of creating too many buckets. + + const sortedAndSlicedErrorGroups = orderBy( errorGroups, (group) => { if (sortField === 'occurrences') { @@ -121,12 +110,68 @@ export async function getServiceErrorGroups({ return group[sortField]; }, [sortDirection] + ).slice(pageIndex * size, pageIndex * size + size); + + const sortedErrorGroupIds = sortedAndSlicedErrorGroups.map( + (group) => group.group_id ); - const sliceFrom = pageIndex * size; + const timeseriesResponse = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [ERROR_GROUP_ID]: sortedErrorGroupIds } }, + { term: { [SERVICE_NAME]: serviceName } }, + { range: rangeFilter(start, end) }, + ...esFilter, + ], + }, + }, + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size, + }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, + }, + }, + }, + }, + }, + }, + }, + }); return { - total_error_groups: sortedErrorGroups.length, - error_groups: sortedErrorGroups.slice(sliceFrom, sliceFrom + size), + total_error_groups: errorGroups.length, + is_aggregation_accurate: + (response.aggregations?.error_groups.sum_other_doc_count ?? 0) === 0, + error_groups: sortedAndSlicedErrorGroups.map((errorGroup) => ({ + ...errorGroup, + occurrences: { + ...errorGroup.occurrences, + timeseries: + timeseriesResponse.aggregations?.error_groups.buckets + .find((bucket) => bucket.key === errorGroup.group_id) + ?.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.doc_count, + })) ?? null, + }, + })), }; } diff --git a/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts index 85a278b07cf7d..b699a30d40418 100644 --- a/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts +++ b/x-pack/test/apm_api_integration/basic/tests/service_overview/error_groups.ts @@ -35,7 +35,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { ); expect(response.status).to.be(200); - expect(response.body).to.eql({ total_error_groups: 0, error_groups: [] }); + expect(response.body).to.eql({ + total_error_groups: 0, + error_groups: [], + is_aggregation_accurate: true, + }); }); }); diff --git a/x-pack/typings/elasticsearch/aggregations.d.ts b/x-pack/typings/elasticsearch/aggregations.d.ts index 29c78e9383175..bc9ed447c8717 100644 --- a/x-pack/typings/elasticsearch/aggregations.d.ts +++ b/x-pack/typings/elasticsearch/aggregations.d.ts @@ -204,6 +204,8 @@ type SubAggregationResponseOf< interface AggregationResponsePart { terms: { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; buckets: Array< { doc_count: number; From 555ba6220694ccf18131f750d500921c2d91557d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 12 Nov 2020 12:19:34 +0100 Subject: [PATCH 3/4] Set fixed width for columns --- .../service_overview/service_overview_errors_table/index.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx index b65286800a769..a5a002cf3aca4 100644 --- a/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_overview/service_overview_errors_table/index.tsx @@ -21,7 +21,7 @@ import { SparkPlotWithValueLabel } from '../../../shared/charts/spark_plot/spark import { callApmApi } from '../../../../services/rest/createCallApmApi'; import { TimestampTooltip } from '../../../shared/TimestampTooltip'; import { ErrorDetailLink } from '../../../shared/Links/apm/ErrorDetailLink'; -import { truncate } from '../../../../style/variables'; +import { px, truncate, unit } from '../../../../style/variables'; import { FetchWrapper } from './fetch_wrapper'; interface Props { @@ -82,7 +82,6 @@ export function ServiceOverviewErrorsTable({ serviceName }: Props) { name: i18n.translate('xpack.apm.serviceOverview.errorsTableColumnName', { defaultMessage: 'Name', }), - width: '60%', render: (_, { name, group_id: errorGroupId }) => { return ( @@ -109,6 +108,7 @@ export function ServiceOverviewErrorsTable({ serviceName }: Props) { render: (_, { last_seen: lastSeen }) => { return ; }, + width: px(unit * 8), }, { field: 'occurrences', @@ -118,6 +118,7 @@ export function ServiceOverviewErrorsTable({ serviceName }: Props) { defaultMessage: 'Occurrences', } ), + width: px(unit * 12), render: (_, { occurrences }) => { return ( Date: Thu, 12 Nov 2020 12:42:48 +0100 Subject: [PATCH 4/4] Update alerting tests to accommodate type updates --- .../monitoring/workload_statistics.test.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts index d1c5256c81c63..c2e62b6e1898b 100644 --- a/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts +++ b/x-pack/plugins/task_manager/server/monitoring/workload_statistics.test.ts @@ -46,9 +46,13 @@ describe('Workload Statistics Aggregator', () => { aggregations: { taskType: { buckets: [], + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, }, schedule: { buckets: [], + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, }, idleTasks: { doc_count: 0, @@ -158,6 +162,8 @@ describe('Workload Statistics Aggregator', () => { }, aggregations: { schedule: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { key: '3600s', @@ -174,11 +180,15 @@ describe('Workload Statistics Aggregator', () => { ], }, taskType: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { key: 'actions_telemetry', doc_count: 2, status: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { key: 'idle', @@ -191,6 +201,8 @@ describe('Workload Statistics Aggregator', () => { key: 'alerting_telemetry', doc_count: 1, status: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { key: 'idle', @@ -203,6 +215,8 @@ describe('Workload Statistics Aggregator', () => { key: 'session_cleanup', doc_count: 1, status: { + doc_count_error_upper_bound: 0, + sum_other_doc_count: 0, buckets: [ { key: 'idle', @@ -608,6 +622,7 @@ describe('padBuckets', () => { key: 1601668047000, doc_count: 1, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -617,6 +632,7 @@ describe('padBuckets', () => { key: 1601668050000, doc_count: 1, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -626,6 +642,7 @@ describe('padBuckets', () => { key: 1601668053000, doc_count: 0, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -635,6 +652,7 @@ describe('padBuckets', () => { key: 1601668056000, doc_count: 0, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -644,6 +662,7 @@ describe('padBuckets', () => { key: 1601668059000, doc_count: 0, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -653,6 +672,7 @@ describe('padBuckets', () => { key: 1601668062000, doc_count: 1, interval: { + doc_count_error_upper_bound: 0, sum_other_doc_count: 0, buckets: [], }, @@ -678,13 +698,13 @@ describe('padBuckets', () => { key_as_string: '2020-10-02T20:40:09.000Z', key: 1601671209000, doc_count: 1, - interval: { buckets: [] }, + interval: { buckets: [], sum_other_doc_count: 0, doc_count_error_upper_bound: 0 }, }, { key_as_string: '2020-10-02T20:40:12.000Z', key: 1601671212000, doc_count: 1, - interval: { buckets: [] }, + interval: { buckets: [], sum_other_doc_count: 0, doc_count_error_upper_bound: 0 }, }, ], }, @@ -707,13 +727,13 @@ describe('padBuckets', () => { key_as_string: '2020-10-02T20:40:09.000Z', key: 1601671209000, doc_count: 1, - interval: { buckets: [] }, + interval: { buckets: [], sum_other_doc_count: 0, doc_count_error_upper_bound: 0 }, }, { key_as_string: '2020-10-02T20:40:12.000Z', key: 1601671212000, doc_count: 1, - interval: { buckets: [] }, + interval: { buckets: [], sum_other_doc_count: 0, doc_count_error_upper_bound: 0 }, }, ], }, @@ -796,7 +816,7 @@ function mockHistogram( key_as_string: key.toISOString(), key: key.getTime(), doc_count: count, - interval: { buckets: [] }, + interval: { buckets: [], doc_count_error_upper_bound: 0, sum_other_doc_count: 0 }, }); } return histogramBuckets; @@ -806,6 +826,8 @@ function mockHistogram( key: number; doc_count: number; interval: { + doc_count_error_upper_bound: number; + sum_other_doc_count: number; buckets: Array<{ key: string; doc_count: number;