[Lens] Support histogram mapping type for all numeric functions (#90357)

* [Lens] Support histogram mapping type * Fix field stats and allow max/min * Fix types * Revert to regular sample data * Simplify server code * Add test for edge case Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
elastic · Feb 16, 2021 · 16d86b0 · 16d86b0
1 parent 5f500a3
commit 16d86b0
Show file tree

Hide file tree

Showing 13 changed files with 256 additions and 28 deletions.
diff --git a/src/plugins/data/common/search/aggs/metrics/max.ts b/src/plugins/data/common/search/aggs/metrics/max.ts
@@ -36,7 +36,7 @@ export const getMaxMetricAgg = () => {
       {
         name: 'field',
         type: 'field',
-        filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE],
+        filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM],
       },
     ],
   });

diff --git a/src/plugins/data/common/search/aggs/metrics/min.ts b/src/plugins/data/common/search/aggs/metrics/min.ts
@@ -36,7 +36,7 @@ export const getMinMetricAgg = () => {
       {
         name: 'field',
         type: 'field',
-        filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE],
+        filterFieldTypes: [KBN_FIELD_TYPES.NUMBER, KBN_FIELD_TYPES.DATE, KBN_FIELD_TYPES.HISTOGRAM],
       },
     ],
   });

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx
@@ -57,7 +57,15 @@ function sortFields(fieldA: IndexPatternField, fieldB: IndexPatternField) {
   return fieldA.displayName.localeCompare(fieldB.displayName, undefined, { sensitivity: 'base' });
 }
 
-const supportedFieldTypes = new Set(['string', 'number', 'boolean', 'date', 'ip', 'document']);
+const supportedFieldTypes = new Set([
+  'string',
+  'number',
+  'boolean',
+  'date',
+  'ip',
+  'histogram',
+  'document',
+]);
 
 const fieldTypeNames: Record<DataType, string> = {
   document: i18n.translate('xpack.lens.datatypes.record', { defaultMessage: 'record' }),
@@ -66,6 +74,7 @@ const fieldTypeNames: Record<DataType, string> = {
   boolean: i18n.translate('xpack.lens.datatypes.boolean', { defaultMessage: 'boolean' }),
   date: i18n.translate('xpack.lens.datatypes.date', { defaultMessage: 'date' }),
   ip: i18n.translate('xpack.lens.datatypes.ipAddress', { defaultMessage: 'IP' }),
+  histogram: i18n.translate('xpack.lens.datatypes.histogram', { defaultMessage: 'histogram' }),
 };
 
 // Wrapper around esQuery.buildEsQuery, handling errors (e.g. because a query can't be parsed) by

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx
@@ -32,6 +32,8 @@ const typeToFn: Record<string, string> = {
   median: 'aggMedian',
 };
 
+const supportedTypes = ['number', 'histogram'];
+
 function buildMetricOperation<T extends MetricColumn<string>>({
   type,
   displayName,
@@ -61,7 +63,7 @@ function buildMetricOperation<T extends MetricColumn<string>>({
     timeScalingMode: optionalTimeScaling ? 'optional' : undefined,
     getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
       if (
-        fieldType === 'number' &&
+        supportedTypes.includes(fieldType) &&
         aggregatable &&
         (!aggregationRestrictions || aggregationRestrictions[type])
       ) {
@@ -77,7 +79,7 @@ function buildMetricOperation<T extends MetricColumn<string>>({
 
       return Boolean(
         newField &&
-          newField.type === 'number' &&
+          supportedTypes.includes(newField.type) &&
           newField.aggregatable &&
           (!newField.aggregationRestrictions || newField.aggregationRestrictions![type])
       );

diff --git a/...ck/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx b/...ck/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.test.tsx
@@ -68,6 +68,52 @@ describe('percentile', () => {
     };
   });
 
+  describe('getPossibleOperationForField', () => {
+    it('should accept number', () => {
+      expect(
+        percentileOperation.getPossibleOperationForField({
+          name: 'bytes',
+          displayName: 'bytes',
+          type: 'number',
+          esTypes: ['long'],
+          aggregatable: true,
+        })
+      ).toEqual({
+        dataType: 'number',
+        isBucketed: false,
+        scale: 'ratio',
+      });
+    });
+
+    it('should accept histogram', () => {
+      expect(
+        percentileOperation.getPossibleOperationForField({
+          name: 'response_time',
+          displayName: 'response_time',
+          type: 'histogram',
+          esTypes: ['histogram'],
+          aggregatable: true,
+        })
+      ).toEqual({
+        dataType: 'number',
+        isBucketed: false,
+        scale: 'ratio',
+      });
+    });
+
+    it('should reject keywords', () => {
+      expect(
+        percentileOperation.getPossibleOperationForField({
+          name: 'origin',
+          displayName: 'origin',
+          type: 'string',
+          esTypes: ['keyword'],
+          aggregatable: true,
+        })
+      ).toBeUndefined();
+    });
+  });
+
   describe('toEsAggsFn', () => {
     it('should reflect params correctly', () => {
       const percentileColumn = layer.columns.col2 as PercentileIndexPatternColumn;
@@ -134,6 +180,34 @@ describe('percentile', () => {
     });
   });
 
+  describe('isTransferable', () => {
+    it('should transfer from number to histogram', () => {
+      const indexPattern = createMockedIndexPattern();
+      indexPattern.getFieldByName = jest.fn().mockReturnValue({
+        name: 'response_time',
+        displayName: 'response_time',
+        type: 'histogram',
+        esTypes: ['histogram'],
+        aggregatable: true,
+      });
+      expect(
+        percentileOperation.isTransferable(
+          {
+            label: '',
+            sourceField: 'response_time',
+            isBucketed: false,
+            dataType: 'number',
+            operationType: 'percentile',
+            params: {
+              percentile: 95,
+            },
+          },
+          indexPattern
+        )
+      ).toBeTruthy();
+    });
+  });
+
   describe('param editor', () => {
     it('should render current percentile', () => {
       const updateLayerSpy = jest.fn();

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/percentile.tsx
@@ -42,14 +42,16 @@ function ofName(name: string, percentile: number) {
 
 const DEFAULT_PERCENTILE_VALUE = 95;
 
+const supportedFieldTypes = ['number', 'histogram'];
+
 export const percentileOperation: OperationDefinition<PercentileIndexPatternColumn, 'field'> = {
   type: 'percentile',
   displayName: i18n.translate('xpack.lens.indexPattern.percentile', {
     defaultMessage: 'Percentile',
   }),
   input: 'field',
   getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type: fieldType }) => {
-    if (fieldType === 'number' && aggregatable && !aggregationRestrictions) {
+    if (supportedFieldTypes.includes(fieldType) && aggregatable && !aggregationRestrictions) {
       return {
         dataType: 'number',
         isBucketed: false,
@@ -62,7 +64,7 @@ export const percentileOperation: OperationDefinition<PercentileIndexPatternColu
 
     return Boolean(
       newField &&
-        newField.type === 'number' &&
+        supportedFieldTypes.includes(newField.type) &&
         newField.aggregatable &&
         !newField.aggregationRestrictions
     );

diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts
@@ -21,6 +21,7 @@ import { getInvalidFieldMessage } from './operations/definitions/helpers';
  * produce 'number')
  */
 export function normalizeOperationDataType(type: DataType) {
+  if (type === 'histogram') return 'number';
   return type === 'document' ? 'number' : type;
 }
 

diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts
@@ -318,7 +318,8 @@ export type DatasourceDimensionDropHandlerProps<T> = DatasourceDimensionDropProp
   dropType: DropType;
 };
 
-export type DataType = 'document' | 'string' | 'number' | 'date' | 'boolean' | 'ip';
+export type FieldOnlyDataType = 'document' | 'ip' | 'histogram';
+export type DataType = 'string' | 'number' | 'date' | 'boolean' | FieldOnlyDataType;
 
 // An operation represents a column in a table, not any information
 // about how the column was created such as whether it is a sum or average.

diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts
@@ -26,6 +26,7 @@ const columnSortOrder = {
   ip: 3,
   boolean: 4,
   number: 5,
+  histogram: 6,
 };
 
 /**

diff --git a/x-pack/plugins/lens/server/routes/field_stats.ts b/x-pack/plugins/lens/server/routes/field_stats.ts
@@ -86,7 +86,11 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
           return result;
         };
 
-        if (field.type === 'number') {
+        if (field.type === 'histogram') {
+          return res.ok({
+            body: await getNumberHistogram(search, field, false),
+          });
+        } else if (field.type === 'number') {
           return res.ok({
             body: await getNumberHistogram(search, field),
           });
@@ -120,36 +124,50 @@ export async function initFieldsRoute(setup: CoreSetup<PluginStartContract>) {
 
 export async function getNumberHistogram(
   aggSearchWithBody: (body: unknown) => Promise<unknown>,
-  field: IFieldType
+  field: IFieldType,
+  useTopHits = true
 ): Promise<FieldStatsResponse> {
   const fieldRef = getFieldRef(field);
 
-  const searchBody = {
+  const baseAggs = {
+    min_value: {
+      min: { field: field.name },
+    },
+    max_value: {
+      max: { field: field.name },
+    },
+    sample_count: { value_count: { ...fieldRef } },
+  };
+  const searchWithoutHits = {
+    sample: {
+      sampler: { shard_size: SHARD_SIZE },
+      aggs: { ...baseAggs },
+    },
+  };
+  const searchWithHits = {
     sample: {
       sampler: { shard_size: SHARD_SIZE },
       aggs: {
-        min_value: {
-          min: { field: field.name },
-        },
-        max_value: {
-          max: { field: field.name },
-        },
-        sample_count: { value_count: { ...fieldRef } },
+        ...baseAggs,
         top_values: {
           terms: { ...fieldRef, size: 10 },
         },
       },
     },
   };
 
-  const minMaxResult = (await aggSearchWithBody(searchBody)) as ESSearchResponse<
-    unknown,
-    { body: { aggs: typeof searchBody } }
-  >;
+  const minMaxResult = (await aggSearchWithBody(
+    useTopHits ? searchWithHits : searchWithoutHits
+  )) as
+    | ESSearchResponse<unknown, { body: { aggs: typeof searchWithHits } }>
+    | ESSearchResponse<unknown, { body: { aggs: typeof searchWithoutHits } }>;
 
   const minValue = minMaxResult.aggregations!.sample.min_value.value;
   const maxValue = minMaxResult.aggregations!.sample.max_value.value;
-  const terms = minMaxResult.aggregations!.sample.top_values;
+  const terms =
+    'top_values' in minMaxResult.aggregations!.sample
+      ? minMaxResult.aggregations!.sample.top_values
+      : { buckets: [] };
   const topValuesBuckets = {
     buckets: terms.buckets.map((bucket) => ({
       count: bucket.doc_count,
@@ -169,7 +187,12 @@ export async function getNumberHistogram(
       sampledValues: minMaxResult.aggregations!.sample.sample_count.value!,
       sampledDocuments: minMaxResult.aggregations!.sample.doc_count,
       topValues: topValuesBuckets,
-      histogram: { buckets: [] },
+      histogram: useTopHits
+        ? { buckets: [] }
+        : {
+            // Insert a fake bucket for a single-value histogram
+            buckets: [{ count: minMaxResult.aggregations!.sample.doc_count, key: minValue }],
+          },
     };
   }