diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/__test__/distribution.test.ts b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/__test__/distribution.test.ts index d944e3fa6db7c..1586e1f4903a2 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/__test__/distribution.test.ts +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/__test__/distribution.test.ts @@ -5,8 +5,6 @@ */ import { getFormattedBuckets } from '../index'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { IBucket } from '../../../../../../server/lib/transactions/distribution/get_buckets/transform'; describe('Distribution', () => { it('getFormattedBuckets', () => { @@ -20,6 +18,7 @@ describe('Distribution', () => { samples: [ { transactionId: 'someTransactionId', + traceId: 'someTraceId', }, ], }, @@ -29,10 +28,12 @@ describe('Distribution', () => { samples: [ { transactionId: 'anotherTransactionId', + traceId: 'anotherTraceId', }, ], }, - ] as IBucket[]; + ]; + expect(getFormattedBuckets(buckets, 20)).toEqual([ { x: 20, x0: 0, y: 0, style: { cursor: 'default' } }, { x: 40, x0: 20, y: 0, style: { cursor: 'default' } }, diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx index fa5a9956c8287..e18e6b1ed914e 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/Distribution/index.tsx @@ -13,7 +13,7 @@ import { ValuesType } from 'utility-types'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { TransactionDistributionAPIResponse } from '../../../../../server/lib/transactions/distribution'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { IBucket } from '../../../../../server/lib/transactions/distribution/get_buckets/transform'; +import { DistributionBucket } from '../../../../../server/lib/transactions/distribution/get_buckets'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { getDurationFormatter } from '../../../../utils/formatters'; // @ts-expect-error @@ -30,7 +30,10 @@ interface IChartPoint { }; } -export function getFormattedBuckets(buckets: IBucket[], bucketSize: number) { +export function getFormattedBuckets( + buckets: DistributionBucket[], + bucketSize: number +) { if (!buckets) { return []; } diff --git a/x-pack/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/index.tsx b/x-pack/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/index.tsx index 0845e2c9b1412..3bb23fd6396ca 100644 --- a/x-pack/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/index.tsx +++ b/x-pack/plugins/apm/public/components/app/TransactionDetails/WaterfallWithSummmary/index.tsx @@ -18,7 +18,7 @@ import { Location } from 'history'; import React, { useEffect, useState } from 'react'; import { useHistory } from 'react-router-dom'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { IBucket } from '../../../../../server/lib/transactions/distribution/get_buckets/transform'; +import { DistributionBucket } from '../../../../../server/lib/transactions/distribution/get_buckets'; import { IUrlParams } from '../../../../context/UrlParamsContext/types'; import { fromQuery, toQuery } from '../../../shared/Links/url_helpers'; import { LoadingStatePrompt } from '../../../shared/LoadingStatePrompt'; @@ -34,7 +34,7 @@ interface Props { waterfall: IWaterfall; exceedsMax: boolean; isLoading: boolean; - traceSamples: IBucket['samples']; + traceSamples: DistributionBucket['samples']; } export function WaterfallWithSummmary({ diff --git a/x-pack/plugins/apm/public/hooks/useTransactionDistribution.ts b/x-pack/plugins/apm/public/hooks/useTransactionDistribution.ts index 65482c9d21c16..1143639f10f47 100644 --- a/x-pack/plugins/apm/public/hooks/useTransactionDistribution.ts +++ b/x-pack/plugins/apm/public/hooks/useTransactionDistribution.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { flatten, omit } from 'lodash'; +import { flatten, omit, isEmpty } from 'lodash'; import { useHistory, useParams } from 'react-router-dom'; import { IUrlParams } from '../context/UrlParamsContext/types'; import { useFetcher } from './useFetcher'; @@ -69,10 +69,12 @@ export function useTransactionDistribution(urlParams: IUrlParams) { // selected sample was not found. select a new one: // sorted by total number of requests, but only pick // from buckets that have samples + const bucketsSortedByPreference = response.buckets + .filter((bucket) => !isEmpty(bucket.samples)) + .sort((bucket) => bucket.count); + const preferredSample = maybe( - response.buckets - .filter((bucket) => bucket.samples.length > 0) - .sort((bucket) => bucket.count)[0]?.samples[0] + bucketsSortedByPreference[0]?.samples[0] ); history.push({ diff --git a/x-pack/plugins/apm/server/lib/transactions/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/transactions/__snapshots__/queries.test.ts.snap index e532265de24ec..c63dfcc0c0ec7 100644 --- a/x-pack/plugins/apm/server/lib/transactions/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/transactions/__snapshots__/queries.test.ts.snap @@ -639,7 +639,7 @@ Object { "body": Object { "aggs": Object { "stats": Object { - "extended_stats": Object { + "max": Object { "field": "transaction.duration.us", }, }, diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/fetcher.ts deleted file mode 100644 index bfe72bf7c00f9..0000000000000 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/fetcher.ts +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { ProcessorEvent } from '../../../../../common/processor_event'; -import { - SERVICE_NAME, - TRACE_ID, - TRANSACTION_DURATION, - TRANSACTION_ID, - TRANSACTION_NAME, - TRANSACTION_SAMPLED, - TRANSACTION_TYPE, -} from '../../../../../common/elasticsearch_fieldnames'; -import { rangeFilter } from '../../../../../common/utils/range_filter'; -import { - Setup, - SetupTimeRange, - SetupUIFilters, -} from '../../../helpers/setup_request'; - -export async function bucketFetcher( - serviceName: string, - transactionName: string, - transactionType: string, - transactionId: string, - traceId: string, - distributionMax: number, - bucketSize: number, - setup: Setup & SetupTimeRange & SetupUIFilters -) { - const { start, end, uiFiltersES, apmEventClient } = setup; - - const params = { - apm: { - events: [ProcessorEvent.transaction as const], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - { term: { [TRANSACTION_NAME]: transactionName } }, - { range: rangeFilter(start, end) }, - ...uiFiltersES, - ], - should: [ - { term: { [TRACE_ID]: traceId } }, - { term: { [TRANSACTION_ID]: transactionId } }, - ], - }, - }, - aggs: { - distribution: { - histogram: { - field: TRANSACTION_DURATION, - interval: bucketSize, - min_doc_count: 0, - extended_bounds: { - min: 0, - max: distributionMax, - }, - }, - aggs: { - samples: { - filter: { - term: { [TRANSACTION_SAMPLED]: true }, - }, - aggs: { - items: { - top_hits: { - _source: [TRANSACTION_ID, TRACE_ID], - size: 10, - }, - }, - }, - }, - }, - }, - }, - }, - }; - - const response = await apmEventClient.search(params); - - return response; -} diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts index f3788b5d812a8..6e2fe34a5f5ef 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts @@ -3,35 +3,204 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - +import { ValuesType } from 'utility-types'; +import { PromiseReturnType } from '../../../../../typings/common'; +import { joinByKey } from '../../../../../common/utils/join_by_key'; +import { ProcessorEvent } from '../../../../../common/processor_event'; +import { + SERVICE_NAME, + TRACE_ID, + TRANSACTION_DURATION, + TRANSACTION_ID, + TRANSACTION_NAME, + TRANSACTION_SAMPLED, + TRANSACTION_TYPE, +} from '../../../../../common/elasticsearch_fieldnames'; +import { rangeFilter } from '../../../../../common/utils/range_filter'; import { Setup, SetupTimeRange, SetupUIFilters, } from '../../../helpers/setup_request'; -import { bucketFetcher } from './fetcher'; -import { bucketTransformer } from './transform'; - -export async function getBuckets( - serviceName: string, - transactionName: string, - transactionType: string, - transactionId: string, - traceId: string, - distributionMax: number, - bucketSize: number, - setup: Setup & SetupTimeRange & SetupUIFilters -) { - const response = await bucketFetcher( - serviceName, - transactionName, - transactionType, - transactionId, - traceId, - distributionMax, - bucketSize, - setup - ); +import { + getDocumentTypeFilterForAggregatedTransactions, + getProcessorEventForAggregatedTransactions, + getTransactionDurationFieldForAggregatedTransactions, +} from '../../../helpers/aggregated_transactions'; + +function getHistogramAggOptions({ + bucketSize, + field, + distributionMax, +}: { + bucketSize: number; + field: string; + distributionMax: number; +}) { + return { + field, + interval: bucketSize, + min_doc_count: 0, + extended_bounds: { + min: 0, + max: distributionMax, + }, + }; +} + +export async function getBuckets({ + serviceName, + transactionName, + transactionType, + transactionId, + traceId, + distributionMax, + bucketSize, + setup, + searchAggregatedTransactions, +}: { + serviceName: string; + transactionName: string; + transactionType: string; + transactionId: string; + traceId: string; + distributionMax: number; + bucketSize: number; + setup: Setup & SetupTimeRange & SetupUIFilters; + searchAggregatedTransactions: boolean; +}) { + const { start, end, uiFiltersES, apmEventClient } = setup; + + const commonFilters = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + { term: { [TRANSACTION_NAME]: transactionName } }, + { range: rangeFilter(start, end) }, + ...uiFiltersES, + ]; - return bucketTransformer(response); + async function getSamplesForDistributionBuckets() { + const response = await apmEventClient.search({ + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + { term: { [TRANSACTION_SAMPLED]: true } }, + ], + should: [ + { term: { [TRACE_ID]: traceId } }, + { term: { [TRANSACTION_ID]: transactionId } }, + ], + }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + bucketSize, + field: TRANSACTION_DURATION, + distributionMax, + }), + aggs: { + samples: { + top_hits: { + _source: [TRANSACTION_ID, TRACE_ID], + size: 10, + sort: { + _score: 'desc', + }, + }, + }, + }, + }, + }, + }, + }); + + return ( + response.aggregations?.distribution.buckets.map((bucket) => { + return { + key: bucket.key, + samples: bucket.samples.hits.hits.map((hit) => ({ + traceId: hit._source.trace.id, + transactionId: hit._source.transaction.id, + })), + }; + }) ?? [] + ); + } + + async function getDistributionBuckets() { + const response = await apmEventClient.search({ + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), + bucketSize, + distributionMax, + }), + }, + }, + }, + }); + + return ( + response.aggregations?.distribution.buckets.map((bucket) => { + return { + key: bucket.key, + count: bucket.doc_count, + }; + }) ?? [] + ); + } + + const [ + samplesForDistributionBuckets, + distributionBuckets, + ] = await Promise.all([ + getSamplesForDistributionBuckets(), + getDistributionBuckets(), + ]); + + const buckets = joinByKey( + [...samplesForDistributionBuckets, ...distributionBuckets], + 'key' + ).map((bucket) => ({ + ...bucket, + samples: bucket.samples ?? [], + count: bucket.count ?? 0, + })); + + return { + noHits: buckets.length === 0, + bucketSize, + buckets, + }; } + +export type DistributionBucket = ValuesType< + PromiseReturnType['buckets'] +>; diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/transform.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/transform.ts deleted file mode 100644 index aee93e34f93a3..0000000000000 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/transform.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { PromiseReturnType } from '../../../../../../observability/typings/common'; -import { Transaction } from '../../../../../typings/es_schemas/ui/transaction'; -import { bucketFetcher } from './fetcher'; - -type DistributionBucketResponse = PromiseReturnType; - -export type IBucket = ReturnType; - -function getBucket( - bucket: Required< - DistributionBucketResponse - >['aggregations']['distribution']['buckets'][0] -) { - const samples = bucket.samples.items.hits.hits.map( - ({ _source }: { _source: Transaction }) => ({ - traceId: _source.trace.id, - transactionId: _source.transaction.id, - }) - ); - - return { - key: bucket.key, - count: bucket.doc_count, - samples, - }; -} - -export function bucketTransformer(response: DistributionBucketResponse) { - const buckets = - response.aggregations?.distribution.buckets.map(getBucket) || []; - - return { - noHits: response.hits.total.value === 0, - buckets, - }; -} diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts index 139dac3df1171..24ca2a4a07b68 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts @@ -4,10 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ProcessorEvent } from '../../../../common/processor_event'; import { SERVICE_NAME, - TRANSACTION_DURATION, TRANSACTION_NAME, TRANSACTION_TYPE, } from '../../../../common/elasticsearch_fieldnames'; @@ -16,18 +14,33 @@ import { SetupTimeRange, SetupUIFilters, } from '../../helpers/setup_request'; +import { + getProcessorEventForAggregatedTransactions, + getTransactionDurationFieldForAggregatedTransactions, +} from '../../helpers/aggregated_transactions'; -export async function getDistributionMax( - serviceName: string, - transactionName: string, - transactionType: string, - setup: Setup & SetupTimeRange & SetupUIFilters -) { +export async function getDistributionMax({ + serviceName, + transactionName, + transactionType, + setup, + searchAggregatedTransactions, +}: { + serviceName: string; + transactionName: string; + transactionType: string; + setup: Setup & SetupTimeRange & SetupUIFilters; + searchAggregatedTransactions: boolean; +}) { const { start, end, uiFiltersES, apmEventClient } = setup; const params = { apm: { - events: [ProcessorEvent.transaction], + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], }, body: { size: 0, @@ -52,8 +65,10 @@ export async function getDistributionMax( }, aggs: { stats: { - extended_stats: { - field: TRANSACTION_DURATION, + max: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, }, }, @@ -61,5 +76,5 @@ export async function getDistributionMax( }; const resp = await apmEventClient.search(params); - return resp.aggregations ? resp.aggregations.stats.max : null; + return resp.aggregations?.stats.value ?? null; } diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/index.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/index.ts index 7c02852f83768..b9ab36fb08d42 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/index.ts @@ -32,6 +32,7 @@ export async function getTransactionDistribution({ transactionId, traceId, setup, + searchAggregatedTransactions, }: { serviceName: string; transactionName: string; @@ -39,20 +40,23 @@ export async function getTransactionDistribution({ transactionId: string; traceId: string; setup: Setup & SetupTimeRange & SetupUIFilters; + searchAggregatedTransactions: boolean; }) { - const distributionMax = await getDistributionMax( + const distributionMax = await getDistributionMax({ serviceName, transactionName, transactionType, - setup - ); + setup, + searchAggregatedTransactions, + }); if (distributionMax == null) { return { noHits: true, buckets: [], bucketSize: 0 }; } const bucketSize = getBucketSize(distributionMax); - const { buckets, noHits } = await getBuckets( + + const { buckets, noHits } = await getBuckets({ serviceName, transactionName, transactionType, @@ -60,8 +64,9 @@ export async function getTransactionDistribution({ traceId, distributionMax, bucketSize, - setup - ); + setup, + searchAggregatedTransactions, + }); return { noHits, diff --git a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts index 1c1d30c2d4d6d..87b8bc7c4ae90 100644 --- a/x-pack/plugins/apm/server/lib/transactions/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/transactions/queries.test.ts @@ -102,6 +102,7 @@ describe('transaction queries', () => { traceId: 'qux', transactionId: 'quz', setup, + searchAggregatedTransactions: false, }) ); diff --git a/x-pack/plugins/apm/server/routes/transaction_groups.ts b/x-pack/plugins/apm/server/routes/transaction_groups.ts index 10e917f385e71..dd1335fb2c2a1 100644 --- a/x-pack/plugins/apm/server/routes/transaction_groups.ts +++ b/x-pack/plugins/apm/server/routes/transaction_groups.ts @@ -124,6 +124,10 @@ export const transactionGroupsDistributionRoute = createRoute(() => ({ traceId = '', } = context.params.query; + const searchAggregatedTransactions = await getSearchAggregatedTransactions( + setup + ); + return getTransactionDistribution({ serviceName, transactionType, @@ -131,6 +135,7 @@ export const transactionGroupsDistributionRoute = createRoute(() => ({ transactionId, traceId, setup, + searchAggregatedTransactions, }); }, })); diff --git a/x-pack/test/apm_api_integration/basic/tests/index.ts b/x-pack/test/apm_api_integration/basic/tests/index.ts index 9e1cb1f5872f1..19dd82d617bd9 100644 --- a/x-pack/test/apm_api_integration/basic/tests/index.ts +++ b/x-pack/test/apm_api_integration/basic/tests/index.ts @@ -45,6 +45,7 @@ export default function apmApiIntegrationTests({ loadTestFile }: FtrProviderCont loadTestFile(require.resolve('./transaction_groups/transaction_charts')); loadTestFile(require.resolve('./transaction_groups/error_rate')); loadTestFile(require.resolve('./transaction_groups/breakdown')); + loadTestFile(require.resolve('./transaction_groups/distribution')); }); describe('Observability overview', function () { diff --git a/x-pack/test/apm_api_integration/basic/tests/transaction_groups/distribution.ts b/x-pack/test/apm_api_integration/basic/tests/transaction_groups/distribution.ts new file mode 100644 index 0000000000000..61dc6ea63c252 --- /dev/null +++ b/x-pack/test/apm_api_integration/basic/tests/transaction_groups/distribution.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import expect from '@kbn/expect'; +import qs from 'querystring'; +import { isEmpty } from 'lodash'; +import archives_metadata from '../../../common/archives_metadata'; +import { expectSnapshot } from '../../../common/match_snapshot'; +import { FtrProviderContext } from '../../../common/ftr_provider_context'; + +export default function ApiTest({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + + const archiveName = 'apm_8.0.0'; + const metadata = archives_metadata[archiveName]; + + // url parameters + const { start, end } = metadata; + const uiFilters = {}; + + const url = `/api/apm/services/opbeans-java/transaction_groups/distribution?${qs.stringify({ + start, + end, + uiFilters, + transactionName: 'APIRestController#stats', + transactionType: 'request', + })}`; + + describe('Transaction groups distribution', () => { + describe('when data is not loaded ', () => { + it('handles empty state', async () => { + const response = await supertest.get(url); + + expect(response.status).to.be(200); + + expect(response.body.noHits).to.be(true); + expect(response.body.buckets.length).to.be(0); + }); + }); + + describe('when data is loaded', () => { + let response: any; + before(async () => { + await esArchiver.load(archiveName); + response = await supertest.get(url); + }); + after(() => esArchiver.unload(archiveName)); + + it('returns the correct metadata', () => { + expect(response.status).to.be(200); + expect(response.body.noHits).to.be(false); + expect(response.body.buckets.length).to.be.greaterThan(0); + }); + + it('returns groups with some hits', () => { + expect(response.body.buckets.some((bucket: any) => bucket.count > 0)).to.be(true); + }); + + it('returns groups with some samples', () => { + expect(response.body.buckets.some((bucket: any) => !isEmpty(bucket.samples))).to.be(true); + }); + + it('returns the correct number of buckets', () => { + expectSnapshot(response.body.buckets.length).toMatchInline(`19`); + }); + + it('returns the correct bucket size', () => { + expectSnapshot(response.body.bucketSize).toMatchInline(`1000`); + }); + + it('returns the correct buckets', () => { + const bucketWithSamples = response.body.buckets.find( + (bucket: any) => !isEmpty(bucket.samples) + ); + + expectSnapshot(bucketWithSamples.count).toMatchInline(`2`); + + expectSnapshot(bucketWithSamples.samples.sort((sample: any) => sample.traceId)) + .toMatchInline(` + Array [ + Object { + "traceId": "a1333547d1257c636154290cddd38c3a", + "transactionId": "3e656b390989133d", + }, + Object { + "traceId": "c799c34f4ee2b0f9998745ea7354d599", + "transactionId": "69b6251b239abb46", + }, + ] + `); + }); + }); + }); +}