diff --git a/src/plugins/data/common/search/aggs/metrics/max.ts b/src/plugins/data/common/search/aggs/metrics/max.ts index ee2d5ad03ce3a..5a41cdbb256c8 100644 --- a/src/plugins/data/common/search/aggs/metrics/max.ts +++ b/src/plugins/data/common/search/aggs/metrics/max.ts @@ -36,7 +36,7 @@ export const getMaxMetricAgg = () => { { name: 'field', type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], }, ], }); diff --git a/src/plugins/data/common/search/aggs/metrics/min.ts b/src/plugins/data/common/search/aggs/metrics/min.ts index f9e3c5b59d586..1805546a9fa34 100644 --- a/src/plugins/data/common/search/aggs/metrics/min.ts +++ b/src/plugins/data/common/search/aggs/metrics/min.ts @@ -36,7 +36,7 @@ export const getMinMetricAgg = () => { { name: 'field', type: 'field', - filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE], + filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM], }, ], }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index 8047807093eef..e487e185a8c8f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -57,7 +57,15 @@ function sortFields(fieldA: IndexPatternField, fieldB: IndexPatternField) { return fieldA.displayName.localeCompare(fieldB.displayName, undefined, { sensitivity: 'base' }); } -const supportedFieldTypes = new Set(['string', 'number', 'boolean', 'date', 'ip', 'document']); +const supportedFieldTypes = new Set([ + 'string', + 'number', + 'boolean', + 'date', + 'ip', + 'histogram', + 'document', +]); const fieldTypeNames: Record = { document: i18n.translate('xpack.lens.datatypes.record', { defaultMessage: 'record' }), @@ -66,6 +74,7 @@ const fieldTypeNames: Record = { boolean: i18n.translate('xpack.lens.datatypes.boolean', { defaultMessage: 'boolean' }), date: i18n.translate('xpack.lens.datatypes.date', { defaultMessage: 'date' }), ip: i18n.translate('xpack.lens.datatypes.ipAddress', { defaultMessage: 'IP' }), + histogram: i18n.translate('xpack.lens.datatypes.histogram', { defaultMessage: 'histogram' }), }; // Wrapper around esQuery.buildEsQuery, handling errors (e.g. because a query can't be parsed) by diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index e11ee580deb9b..e724a34be20e8 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -32,6 +32,8 @@ const typeToFn: Record = { median: 'aggMedian', }; +const supportedTypes = ['number', 'histogram']; + function buildMetricOperation>({ type, displayName, @@ -61,7 +63,7 @@ function buildMetricOperation>({ timeScalingMode: optionalTimeScaling ? 'optional' : undefined, getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => { if ( - fieldType === 'number' && + supportedTypes.includes(fieldType) && aggregatable && (!aggregationRestrictions || aggregationRestrictions[type]) ) { @@ -77,7 +79,7 @@ function buildMetricOperation>({ return Boolean( newField && - newField.type === 'number' && + supportedTypes.includes(newField.type) && newField.aggregatable && (!newField.aggregationRestrictions || newField.aggregationRestrictions![type]) ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx index 07bab16b7096f..9ac91be5a17ec 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx @@ -68,6 +68,52 @@ describe('percentile', () => { }; }); + describe('getPossibleOperationForField', () => { + it('should accept number', () => { + expect( + percentileOperation.getPossibleOperationForField({ + name: 'bytes', + displayName: 'bytes', + type: 'number', + esTypes: ['long'], + aggregatable: true, + }) + ).toEqual({ + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }); + }); + + it('should accept histogram', () => { + expect( + percentileOperation.getPossibleOperationForField({ + name: 'response_time', + displayName: 'response_time', + type: 'histogram', + esTypes: ['histogram'], + aggregatable: true, + }) + ).toEqual({ + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }); + }); + + it('should reject keywords', () => { + expect( + percentileOperation.getPossibleOperationForField({ + name: 'origin', + displayName: 'origin', + type: 'string', + esTypes: ['keyword'], + aggregatable: true, + }) + ).toBeUndefined(); + }); + }); + describe('toEsAggsFn', () => { it('should reflect params correctly', () => { const percentileColumn = layer.columns.col2 as PercentileIndexPatternColumn; @@ -134,6 +180,34 @@ describe('percentile', () => { }); }); + describe('isTransferable', () => { + it('should transfer from number to histogram', () => { + const indexPattern = createMockedIndexPattern(); + indexPattern.getFieldByName = jest.fn().mockReturnValue({ + name: 'response_time', + displayName: 'response_time', + type: 'histogram', + esTypes: ['histogram'], + aggregatable: true, + }); + expect( + percentileOperation.isTransferable( + { + label: '', + sourceField: 'response_time', + isBucketed: false, + dataType: 'number', + operationType: 'percentile', + params: { + percentile: 95, + }, + }, + indexPattern + ) + ).toBeTruthy(); + }); + }); + describe('param editor', () => { it('should render current percentile', () => { const updateLayerSpy = jest.fn(); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx index f236b2932b2d3..e7654380bd85f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx @@ -42,6 +42,8 @@ function ofName(name: string, percentile: number) { const DEFAULT_PERCENTILE_VALUE = 95; +const supportedFieldTypes = ['number', 'histogram']; + export const percentileOperation: OperationDefinition = { type: 'percentile', displayName: i18n.translate('xpack.lens.indexPattern.percentile', { @@ -49,7 +51,7 @@ export const percentileOperation: OperationDefinition { - if (fieldType === 'number' && aggregatable && !aggregationRestrictions) { + if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) { return { dataType: 'number', isBucketed: false, @@ -62,7 +64,7 @@ export const percentileOperation: OperationDefinition = DatasourceDimensionDropProp dropType: DropType; }; -export type DataType = 'document' | 'string' | 'number' | 'date' | 'boolean' | 'ip'; +export type FieldOnlyDataType = 'document' | 'ip' | 'histogram'; +export type DataType = 'string' | 'number' | 'date' | 'boolean' | FieldOnlyDataType; // An operation represents a column in a table, not any information // about how the column was created such as whether it is a sum or average. diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts index 8b121232162aa..772934160a058 100644 --- a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts +++ b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts @@ -26,6 +26,7 @@ const columnSortOrder = { ip: 3, boolean: 4, number: 5, + histogram: 6, }; /** diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts index e1681a74c2951..7fd884755d86d 100644 --- a/x-pack/plugins/lens/server/routes/field_stats.ts +++ b/x-pack/plugins/lens/server/routes/field_stats.ts @@ -86,7 +86,11 @@ export async function initFieldsRoute(setup: CoreSetup) { return result; }; - if (field.type === 'number') { + if (field.type === 'histogram') { + return res.ok({ + body: await getNumberHistogram(search, field, false), + }); + } else if (field.type === 'number') { return res.ok({ body: await getNumberHistogram(search, field), }); @@ -120,21 +124,31 @@ export async function initFieldsRoute(setup: CoreSetup) { export async function getNumberHistogram( aggSearchWithBody: (body: unknown) => Promise, - field: IFieldType + field: IFieldType, + useTopHits = true ): Promise { const fieldRef = getFieldRef(field); - const searchBody = { + const baseAggs = { + min_value: { + min: { field: field.name }, + }, + max_value: { + max: { field: field.name }, + }, + sample_count: { value_count: { ...fieldRef } }, + }; + const searchWithoutHits = { + sample: { + sampler: { shard_size: SHARD_SIZE }, + aggs: { ...baseAggs }, + }, + }; + const searchWithHits = { sample: { sampler: { shard_size: SHARD_SIZE }, aggs: { - min_value: { - min: { field: field.name }, - }, - max_value: { - max: { field: field.name }, - }, - sample_count: { value_count: { ...fieldRef } }, + ...baseAggs, top_values: { terms: { ...fieldRef, size: 10 }, }, @@ -142,14 +156,18 @@ export async function getNumberHistogram( }, }; - const minMaxResult = (await aggSearchWithBody(searchBody)) as ESSearchResponse< - unknown, - { body: { aggs: typeof searchBody } } - >; + const minMaxResult = (await aggSearchWithBody( + useTopHits ? searchWithHits : searchWithoutHits + )) as + | ESSearchResponse + | ESSearchResponse; const minValue = minMaxResult.aggregations!.sample.min_value.value; const maxValue = minMaxResult.aggregations!.sample.max_value.value; - const terms = minMaxResult.aggregations!.sample.top_values; + const terms = + 'top_values' in minMaxResult.aggregations!.sample + ? minMaxResult.aggregations!.sample.top_values + : { buckets: [] }; const topValuesBuckets = { buckets: terms.buckets.map((bucket) => ({ count: bucket.doc_count, @@ -169,7 +187,12 @@ export async function getNumberHistogram( sampledValues: minMaxResult.aggregations!.sample.sample_count.value!, sampledDocuments: minMaxResult.aggregations!.sample.doc_count, topValues: topValuesBuckets, - histogram: { buckets: [] }, + histogram: useTopHits + ? { buckets: [] } + : { + // Insert a fake bucket for a single-value histogram + buckets: [{ count: minMaxResult.aggregations!.sample.doc_count, key: minValue }], + }, }; } diff --git a/x-pack/test/api_integration/apis/lens/field_stats.ts b/x-pack/test/api_integration/apis/lens/field_stats.ts index 2cfce5ef31305..ac4ebb4e5b02c 100644 --- a/x-pack/test/api_integration/apis/lens/field_stats.ts +++ b/x-pack/test/api_integration/apis/lens/field_stats.ts @@ -23,10 +23,12 @@ export default ({ getService }: FtrProviderContext) => { before(async () => { await esArchiver.loadIfNeeded('logstash_functional'); await esArchiver.loadIfNeeded('visualize/default'); + await esArchiver.loadIfNeeded('pre_calculated_histogram'); }); after(async () => { await esArchiver.unload('logstash_functional'); await esArchiver.unload('visualize/default'); + await esArchiver.unload('pre_calculated_histogram'); }); describe('field distribution', () => { @@ -347,6 +349,101 @@ export default ({ getService }: FtrProviderContext) => { }); }); + it('should return an auto histogram for precalculated histograms', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/histogram-test/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match_all: {} }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + field: { + name: 'histogram-content', + type: 'histogram', + }, + }) + .expect(200); + + expect(body).to.eql({ + histogram: { + buckets: [ + { + count: 237, + key: 0, + }, + { + count: 323, + key: 0.47000000000000003, + }, + { + count: 454, + key: 0.9400000000000001, + }, + { + count: 166, + key: 1.4100000000000001, + }, + { + count: 168, + key: 1.8800000000000001, + }, + { + count: 425, + key: 2.35, + }, + { + count: 311, + key: 2.8200000000000003, + }, + { + count: 391, + key: 3.29, + }, + { + count: 406, + key: 3.7600000000000002, + }, + { + count: 324, + key: 4.23, + }, + { + count: 628, + key: 4.7, + }, + ], + }, + sampledDocuments: 7, + sampledValues: 3833, + totalDocuments: 7, + topValues: { buckets: [] }, + }); + }); + + it('should return a single-value histogram when filtering a precalculated histogram', async () => { + const { body } = await supertest + .post('/api/lens/index_stats/histogram-test/field') + .set(COMMON_HEADERS) + .send({ + dslQuery: { match: { 'histogram-title': 'single value' } }, + fromDate: TEST_START_TIME, + toDate: TEST_END_TIME, + field: { + name: 'histogram-content', + type: 'histogram', + }, + }) + .expect(200); + + expect(body).to.eql({ + histogram: { buckets: [{ count: 1, key: 1 }] }, + sampledDocuments: 1, + sampledValues: 1, + totalDocuments: 1, + topValues: { buckets: [] }, + }); + }); + it('should return histograms for scripted date fields', async () => { const { body } = await supertest .post('/api/lens/index_stats/logstash-2015.09.22/field') diff --git a/x-pack/test/functional/apps/visualize/precalculated_histogram.ts b/x-pack/test/functional/apps/visualize/precalculated_histogram.ts index a0c8fa3b21ffd..151c9e981250f 100644 --- a/x-pack/test/functional/apps/visualize/precalculated_histogram.ts +++ b/x-pack/test/functional/apps/visualize/precalculated_histogram.ts @@ -64,7 +64,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('with average aggregation', async () => { const data = await renderTableForAggregation('Average'); - expect(data).to.eql([['2.8510720308359434']]); + expect(data).to.eql([['2.8653795982259327']]); }); it('with median aggregation', async () => { @@ -79,7 +79,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('with sum aggregation', async () => { const data = await renderTableForAggregation('Sum'); - expect(data).to.eql([['11834.800000000001']]); + expect(data).to.eql([['10983']]); }); }); }); diff --git a/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json b/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json index cab1dbdf84483..121a4036aaacd 100644 --- a/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json +++ b/x-pack/test/functional/es_archives/pre_calculated_histogram/data.json @@ -5,8 +5,7 @@ "index": ".kibana", "source": { "index-pattern": { - "title": "histogram-test", - "fields": "[{\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"histogram-content\",\"type\":\"histogram\",\"esTypes\":[\"histogram\"],\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"histogram-title\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" + "title": "histogram-test" }, "type": "index-pattern" } @@ -195,3 +194,22 @@ } } } + +{ + "type": "doc", + "value": { + "id": "5e694159d909d9d99b5e12d1", + "index": "histogram-test", + "source": { + "histogram-title": "single value", + "histogram-content": { + "values": [ + 1 + ], + "counts": [ + 1 + ] + } + } + } +}