From 3bf2864df36d546eb51229d4c75b614085f5f16d Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Tue, 23 Feb 2021 11:56:20 +0100 Subject: [PATCH 1/3] [APM] Include services with only metric documents Closes #92075. --- .../apm/common/utils/as_mutable_array.ts | 12 +++ .../__snapshots__/queries.test.ts.snap | 56 ++++++++++++- .../get_service_transaction_stats.ts | 13 ++- .../get_services_from_metric_documents.ts | 82 +++++++++++++++++++ .../get_services/get_services_items.ts | 42 +++++++--- .../tests/services/top_services.ts | 45 ++++++++++ 6 files changed, 231 insertions(+), 19 deletions(-) create mode 100644 x-pack/plugins/apm/common/utils/as_mutable_array.ts create mode 100644 x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts diff --git a/x-pack/plugins/apm/common/utils/as_mutable_array.ts b/x-pack/plugins/apm/common/utils/as_mutable_array.ts new file mode 100644 index 0000000000000..93442ec19751f --- /dev/null +++ b/x-pack/plugins/apm/common/utils/as_mutable_array.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function asMutableArray>( + arr: T +): T extends Readonly<[...infer U]> ? U : unknown[] { + return arr as any; +} diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 3e68831ee7cba..2819a62c30377 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -109,7 +109,6 @@ Array [ "environments": Object { "terms": Object { "field": "service.environment", - "missing": "", }, }, "outcomes": Object { @@ -193,6 +192,61 @@ Array [ "size": 0, }, }, + Object { + "apm": Object { + "events": Array [ + "metric", + ], + }, + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + }, + }, + "latest": Object { + "top_metrics": Object { + "metrics": Object { + "field": "agent.name", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + Object { + "term": Object { + "service.environment": "test", + }, + }, + ], + }, + }, + "size": 0, + }, + }, ] `; diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index e1f8bca83829c..a03bbc3376259 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -35,14 +35,14 @@ interface AggregationParams { environment?: string; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; + maxNumServices: number; } -const MAX_NUMBER_OF_SERVICES = 500; - export async function getServiceTransactionStats({ environment, setup, searchAggregatedTransactions, + maxNumServices, }: AggregationParams) { return withApmSpan('get_service_transaction_stats', async () => { const { apmEventClient, start, end, esFilter } = setup; @@ -86,7 +86,7 @@ export async function getServiceTransactionStats({ services: { terms: { field: SERVICE_NAME, - size: MAX_NUMBER_OF_SERVICES, + size: maxNumServices, }, aggs: { transactionType: { @@ -98,7 +98,6 @@ export async function getServiceTransactionStats({ environments: { terms: { field: SERVICE_ENVIRONMENT, - missing: '', }, }, sample: { @@ -141,9 +140,9 @@ export async function getServiceTransactionStats({ return { serviceName: bucket.key as string, transactionType: topTransactionTypeBucket.key as string, - environments: topTransactionTypeBucket.environments.buckets - .map((environmentBucket) => environmentBucket.key as string) - .filter(Boolean), + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), agentName: topTransactionTypeBucket.sample.top[0].metrics[ AGENT_NAME ] as AgentName, diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts new file mode 100644 index 0000000000000..1d4ce7bf6c6b5 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { + AGENT_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, +} from '../../../../common/elasticsearch_fieldnames'; +import { environmentQuery, rangeQuery } from '../../../../common/utils/queries'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { withApmSpan } from '../../../utils/with_apm_span'; + +export function getServicesFromMetricDocuments({ + environment, + setup, + maxNumServices, +}: { + setup: Setup & SetupTimeRange; + environment?: string; + maxNumServices: number; +}) { + return withApmSpan('get_services_from_metric_documents', async () => { + const { apmEventClient, start, end, esFilter } = setup; + + const response = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...esFilter, + ], + }, + }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxNumServices, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + latest: { + top_metrics: { + metrics: { field: AGENT_NAME } as const, + sort: { '@timestamp': 'desc' }, + }, + }, + }, + }, + }, + }, + }); + + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map( + (envBucket) => envBucket.key as string + ), + agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, + }; + }) ?? [] + ); + }); +} diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts index c2677af038486..169d1fed461e4 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts @@ -6,15 +6,18 @@ */ import { Logger } from '@kbn/logging'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getServicesProjection } from '../../../projections/services'; import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getHealthStatuses } from './get_health_statuses'; +import { getServicesFromMetricDocuments } from './get_services_from_metric_documents'; import { getServiceTransactionStats } from './get_service_transaction_stats'; export type ServicesItemsSetup = Setup & SetupTimeRange; +const MAX_NUMBER_OF_SERVICES = 500; + export async function getServicesItems({ environment, setup, @@ -29,32 +32,49 @@ export async function getServicesItems({ return withApmSpan('get_services_items', async () => { const params = { environment, - projection: getServicesProjection({ - setup, - searchAggregatedTransactions, - }), setup, searchAggregatedTransactions, + maxNumServices: MAX_NUMBER_OF_SERVICES, }; - const [transactionStats, healthStatuses] = await Promise.all([ + const [ + transactionStats, + servicesFromMetricDocuments, + healthStatuses, + ] = await Promise.all([ getServiceTransactionStats(params), + getServicesFromMetricDocuments(params), getHealthStatuses(params).catch((err) => { logger.error(err); return []; }), ]); - const apmServices = transactionStats.map(({ serviceName }) => serviceName); + const foundServiceNames = transactionStats.map( + ({ serviceName }) => serviceName + ); + + const servicesWithOnlyMetricDocuments = servicesFromMetricDocuments.filter( + ({ serviceName }) => !foundServiceNames.includes(serviceName) + ); + + const allServiceNames = foundServiceNames.concat( + servicesWithOnlyMetricDocuments.map(({ serviceName }) => serviceName) + ); // make sure to exclude health statuses from services // that are not found in APM data const matchedHealthStatuses = healthStatuses.filter(({ serviceName }) => - apmServices.includes(serviceName) + allServiceNames.includes(serviceName) ); - const allMetrics = [...transactionStats, ...matchedHealthStatuses]; - - return joinByKey(allMetrics, 'serviceName'); + return joinByKey( + asMutableArray([ + ...transactionStats, + ...servicesWithOnlyMetricDocuments, + ...matchedHealthStatuses, + ] as const), + 'serviceName' + ); }); } diff --git a/x-pack/test/apm_api_integration/tests/services/top_services.ts b/x-pack/test/apm_api_integration/tests/services/top_services.ts index 3afaec653fcb3..bbd6b58574e8a 100644 --- a/x-pack/test/apm_api_integration/tests/services/top_services.ts +++ b/x-pack/test/apm_api_integration/tests/services/top_services.ts @@ -261,6 +261,51 @@ export default function ApiTest({ getService }: FtrProviderContext) { } ); + registry.when( + 'APM Services Overview with a basic license when data is loaded excluding transaction events', + { config: 'basic', archives: [archiveName] }, + () => { + it('includes services that only report metric data', async () => { + interface Response { + status: number; + body: APIReturnType<'GET /api/apm/services'>; + } + + const [unfilteredResponse, filteredResponse] = await Promise.all([ + supertest.get( + `/api/apm/services?start=${start}&end=${end}&uiFilters={}` + ) as Promise, + supertest.get( + `/api/apm/services?start=${start}&end=${end}&uiFilters=${encodeURIComponent( + `{ "kuery": "not (processor.event:transaction)" }` + )}` + ) as Promise, + ]); + + expect(unfilteredResponse.body.items.length).to.be.greaterThan(0); + + const unfilteredServiceNames = unfilteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + const filteredServiceNames = filteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + expect(unfilteredServiceNames).to.eql(filteredServiceNames); + + expect( + filteredResponse.body.items.every((item) => { + // make sure it did not query transaction data + return isEmpty(item.avgResponseTime); + }) + ).to.be(true); + + expect(filteredResponse.body.items.every((item) => !!item.agentName)).to.be(true); + }); + } + ); + registry.when( 'APM Services overview with a trial license when data is loaded', { config: 'trial', archives: [archiveName] }, From e6437258dbadf604a891b5bfa6ddbacd56a26e51 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 25 Feb 2021 11:29:13 +0100 Subject: [PATCH 2/3] Explain as_mutable_array --- .../apm/common/utils/as_mutable_array.ts | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/x-pack/plugins/apm/common/utils/as_mutable_array.ts b/x-pack/plugins/apm/common/utils/as_mutable_array.ts index 93442ec19751f..ce1d7e607ec4c 100644 --- a/x-pack/plugins/apm/common/utils/as_mutable_array.ts +++ b/x-pack/plugins/apm/common/utils/as_mutable_array.ts @@ -5,6 +5,35 @@ * 2.0. */ +// Sometimes we use `as const` to have a more specific type, +// because TypeScript by default will widen the value type of an +// array literal. Consider the following example: +// +// const filter = [ +// { term: { 'agent.name': 'nodejs' } }, +// { range: { '@timestamp': { gte: 'now-15m ' }} +// ]; + +// The result value type will be: + +// const filter: ({ +// term: { +// 'agent.name'?: string +// }; +// range?: undefined +// } | { +// term?: undefined; +// range: { +// '@timestamp': { +// gte: string +// } +// } +// })[]; + +// This can sometimes leads to issues. In those cases, we can +// use `as const`. However, the Readonly type is not compatible +// with Array. This function returns a mutable version of a type. + export function asMutableArray>( arr: T ): T extends Readonly<[...infer U]> ? U : unknown[] { From 22baa96171e69c859d804bf495a0034ceb30bc3a Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Thu, 25 Feb 2021 14:32:56 +0100 Subject: [PATCH 3/3] Use kuery instead of uiFilters for API tests --- .../apm_api_integration/tests/services/top_services.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/x-pack/test/apm_api_integration/tests/services/top_services.ts b/x-pack/test/apm_api_integration/tests/services/top_services.ts index 0a7ed8cc5fb69..37f7b09e8b7d2 100644 --- a/x-pack/test/apm_api_integration/tests/services/top_services.ts +++ b/x-pack/test/apm_api_integration/tests/services/top_services.ts @@ -266,12 +266,10 @@ export default function ApiTest({ getService }: FtrProviderContext) { } const [unfilteredResponse, filteredResponse] = await Promise.all([ + supertest.get(`/api/apm/services?start=${start}&end=${end}`) as Promise, supertest.get( - `/api/apm/services?start=${start}&end=${end}&uiFilters={}` - ) as Promise, - supertest.get( - `/api/apm/services?start=${start}&end=${end}&uiFilters=${encodeURIComponent( - `{ "kuery": "not (processor.event:transaction)" }` + `/api/apm/services?start=${start}&end=${end}&kuery=${encodeURIComponent( + 'not (processor.event:transaction)' )}` ) as Promise, ]);