diff --git a/x-pack/plugins/apm/common/utils/as_mutable_array.ts b/x-pack/plugins/apm/common/utils/as_mutable_array.ts new file mode 100644 index 0000000000000..ce1d7e607ec4c --- /dev/null +++ b/x-pack/plugins/apm/common/utils/as_mutable_array.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// Sometimes we use `as const` to have a more specific type, +// because TypeScript by default will widen the value type of an +// array literal. Consider the following example: +// +// const filter = [ +// { term: { 'agent.name': 'nodejs' } }, +// { range: { '@timestamp': { gte: 'now-15m ' }} +// ]; + +// The result value type will be: + +// const filter: ({ +// term: { +// 'agent.name'?: string +// }; +// range?: undefined +// } | { +// term?: undefined; +// range: { +// '@timestamp': { +// gte: string +// } +// } +// })[]; + +// This can sometimes leads to issues. In those cases, we can +// use `as const`. However, the Readonly type is not compatible +// with Array. This function returns a mutable version of a type. + +export function asMutableArray>( + arr: T +): T extends Readonly<[...infer U]> ? U : unknown[] { + return arr as any; +} diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index dec5be8da32f4..7e7e073c0d2f6 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -118,7 +118,6 @@ Array [ "environments": Object { "terms": Object { "field": "service.environment", - "missing": "", }, }, "outcomes": Object { @@ -197,6 +196,56 @@ Array [ "size": 0, }, }, + Object { + "apm": Object { + "events": Array [ + "metric", + ], + }, + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + }, + }, + "latest": Object { + "top_metrics": Object { + "metrics": Object { + "field": "agent.name", + }, + "sort": Object { + "@timestamp": "desc", + }, + }, + }, + }, + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + ], + }, + }, + "size": 0, + }, + }, ] `; diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index 5f0302035462c..10c7420d0f3b0 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -40,15 +40,15 @@ interface AggregationParams { kuery?: string; setup: ServicesItemsSetup; searchAggregatedTransactions: boolean; + maxNumServices: number; } -const MAX_NUMBER_OF_SERVICES = 500; - export async function getServiceTransactionStats({ environment, kuery, setup, searchAggregatedTransactions, + maxNumServices, }: AggregationParams) { return withApmSpan('get_service_transaction_stats', async () => { const { apmEventClient, start, end } = setup; @@ -92,7 +92,7 @@ export async function getServiceTransactionStats({ services: { terms: { field: SERVICE_NAME, - size: MAX_NUMBER_OF_SERVICES, + size: maxNumServices, }, aggs: { transactionType: { @@ -104,7 +104,6 @@ export async function getServiceTransactionStats({ environments: { terms: { field: SERVICE_ENVIRONMENT, - missing: '', }, }, sample: { @@ -147,9 +146,9 @@ export async function getServiceTransactionStats({ return { serviceName: bucket.key as string, transactionType: topTransactionTypeBucket.key as string, - environments: topTransactionTypeBucket.environments.buckets - .map((environmentBucket) => environmentBucket.key as string) - .filter(Boolean), + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), agentName: topTransactionTypeBucket.sample.top[0].metrics[ AGENT_NAME ] as AgentName, diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts new file mode 100644 index 0000000000000..cabd44c1e6907 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { + AGENT_NAME, + SERVICE_ENVIRONMENT, + SERVICE_NAME, +} from '../../../../common/elasticsearch_fieldnames'; +import { environmentQuery, kqlQuery, rangeQuery } from '../../../utils/queries'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { Setup, SetupTimeRange } from '../../helpers/setup_request'; +import { withApmSpan } from '../../../utils/with_apm_span'; + +export function getServicesFromMetricDocuments({ + environment, + setup, + maxNumServices, + kuery, +}: { + setup: Setup & SetupTimeRange; + environment?: string; + maxNumServices: number; + kuery?: string; +}) { + return withApmSpan('get_services_from_metric_documents', async () => { + const { apmEventClient, start, end } = setup; + + const response = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, + }, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxNumServices, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + latest: { + top_metrics: { + metrics: { field: AGENT_NAME } as const, + sort: { '@timestamp': 'desc' }, + }, + }, + }, + }, + }, + }, + }); + + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map( + (envBucket) => envBucket.key as string + ), + agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, + }; + }) ?? [] + ); + }); +} diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts index 1ddc7a6583c81..3b9792519d261 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts @@ -6,15 +6,18 @@ */ import { Logger } from '@kbn/logging'; +import { asMutableArray } from '../../../../common/utils/as_mutable_array'; import { joinByKey } from '../../../../common/utils/join_by_key'; -import { getServicesProjection } from '../../../projections/services'; import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getHealthStatuses } from './get_health_statuses'; +import { getServicesFromMetricDocuments } from './get_services_from_metric_documents'; import { getServiceTransactionStats } from './get_service_transaction_stats'; export type ServicesItemsSetup = Setup & SetupTimeRange; +const MAX_NUMBER_OF_SERVICES = 500; + export async function getServicesItems({ environment, kuery, @@ -32,33 +35,49 @@ export async function getServicesItems({ const params = { environment, kuery, - projection: getServicesProjection({ - kuery, - setup, - searchAggregatedTransactions, - }), setup, searchAggregatedTransactions, + maxNumServices: MAX_NUMBER_OF_SERVICES, }; - const [transactionStats, healthStatuses] = await Promise.all([ + const [ + transactionStats, + servicesFromMetricDocuments, + healthStatuses, + ] = await Promise.all([ getServiceTransactionStats(params), + getServicesFromMetricDocuments(params), getHealthStatuses(params).catch((err) => { logger.error(err); return []; }), ]); - const apmServices = transactionStats.map(({ serviceName }) => serviceName); + const foundServiceNames = transactionStats.map( + ({ serviceName }) => serviceName + ); + + const servicesWithOnlyMetricDocuments = servicesFromMetricDocuments.filter( + ({ serviceName }) => !foundServiceNames.includes(serviceName) + ); + + const allServiceNames = foundServiceNames.concat( + servicesWithOnlyMetricDocuments.map(({ serviceName }) => serviceName) + ); // make sure to exclude health statuses from services // that are not found in APM data const matchedHealthStatuses = healthStatuses.filter(({ serviceName }) => - apmServices.includes(serviceName) + allServiceNames.includes(serviceName) ); - const allMetrics = [...transactionStats, ...matchedHealthStatuses]; - - return joinByKey(allMetrics, 'serviceName'); + return joinByKey( + asMutableArray([ + ...transactionStats, + ...servicesWithOnlyMetricDocuments, + ...matchedHealthStatuses, + ] as const), + 'serviceName' + ); }); } diff --git a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx index e1427bc96e7e0..c69c7ebff2b9e 100644 --- a/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx +++ b/x-pack/plugins/infra/public/components/log_stream/log_stream_embeddable.tsx @@ -66,7 +66,7 @@ export class LogStreamEmbeddable extends Embeddable { } const startTimestamp = datemathToEpochMillis(this.input.timeRange.from); - const endTimestamp = datemathToEpochMillis(this.input.timeRange.to); + const endTimestamp = datemathToEpochMillis(this.input.timeRange.to, 'up'); if (!startTimestamp || !endTimestamp) { return; diff --git a/x-pack/test/accessibility/apps/search_profiler.ts b/x-pack/test/accessibility/apps/search_profiler.ts index 7fba45175c831..6559d58be6298 100644 --- a/x-pack/test/accessibility/apps/search_profiler.ts +++ b/x-pack/test/accessibility/apps/search_profiler.ts @@ -15,7 +15,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const a11y = getService('a11y'); const flyout = getService('flyout'); - describe('Accessibility Search Profiler Editor', () => { + // FLAKY: https://github.com/elastic/kibana/issues/91939 + describe.skip('Accessibility Search Profiler Editor', () => { before(async () => { await PageObjects.common.navigateToApp('searchProfiler'); await a11y.testAppSnapshot(); diff --git a/x-pack/test/apm_api_integration/tests/services/top_services.ts b/x-pack/test/apm_api_integration/tests/services/top_services.ts index 3896bc1c6fabc..37f7b09e8b7d2 100644 --- a/x-pack/test/apm_api_integration/tests/services/top_services.ts +++ b/x-pack/test/apm_api_integration/tests/services/top_services.ts @@ -255,6 +255,49 @@ export default function ApiTest({ getService }: FtrProviderContext) { } ); + registry.when( + 'APM Services Overview with a basic license when data is loaded excluding transaction events', + { config: 'basic', archives: [archiveName] }, + () => { + it('includes services that only report metric data', async () => { + interface Response { + status: number; + body: APIReturnType<'GET /api/apm/services'>; + } + + const [unfilteredResponse, filteredResponse] = await Promise.all([ + supertest.get(`/api/apm/services?start=${start}&end=${end}`) as Promise, + supertest.get( + `/api/apm/services?start=${start}&end=${end}&kuery=${encodeURIComponent( + 'not (processor.event:transaction)' + )}` + ) as Promise, + ]); + + expect(unfilteredResponse.body.items.length).to.be.greaterThan(0); + + const unfilteredServiceNames = unfilteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + const filteredServiceNames = filteredResponse.body.items + .map((item) => item.serviceName) + .sort(); + + expect(unfilteredServiceNames).to.eql(filteredServiceNames); + + expect( + filteredResponse.body.items.every((item) => { + // make sure it did not query transaction data + return isEmpty(item.avgResponseTime); + }) + ).to.be(true); + + expect(filteredResponse.body.items.every((item) => !!item.agentName)).to.be(true); + }); + } + ); + registry.when( 'APM Services overview with a trial license when data is loaded', { config: 'trial', archives: [archiveName] }, diff --git a/x-pack/test/functional/apps/uptime/overview.ts b/x-pack/test/functional/apps/uptime/overview.ts index 6c9eb24070d8f..b9c1767e4a8cf 100644 --- a/x-pack/test/functional/apps/uptime/overview.ts +++ b/x-pack/test/functional/apps/uptime/overview.ts @@ -15,7 +15,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { const testSubjects = getService('testSubjects'); - describe('overview page', function () { + // FLAKY: https://github.com/elastic/kibana/issues/89072 + describe.skip('overview page', function () { const DEFAULT_DATE_START = 'Sep 10, 2019 @ 12:40:08.078'; const DEFAULT_DATE_END = 'Sep 11, 2019 @ 19:40:08.078';