[ML] Transforms: Fixes chart histograms for runtime fields. (#93028) (#…

…93203)

Fixes chart histograms for runtime fields. The runtime field configurations were not passed on to the endpoint to fetch the charts data, so charts ended up being empty with a 0 documents legend.

Co-authored-by: Walter Rafelsberger <walter@elastic.co>

x-pack/plugins/ml/common/index.ts

@@ -6,6 +6,7 @@
  */
 
 export { HitsTotalRelation, SearchResponse7, HITS_TOTAL_RELATION } from './types/es_client';
+export { ChartData } from './types/field_histograms';
 export { ANOMALY_SEVERITY, ANOMALY_THRESHOLD, SEVERITY_COLORS } from './constants/anomalies';
 export { getSeverityColor, getSeverityType } from './util/anomaly_utils';
 export { composeValidators, patternValidator } from './util/validators';

x-pack/plugins/ml/common/types/field_histograms.ts

@@ -0,0 +1,68 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface NumericDataItem {
+  key: number;
+  key_as_string?: string | number;
+  doc_count: number;
+}
+
+export interface NumericChartData {
+  data: NumericDataItem[];
+  id: string;
+  interval: number;
+  stats: [number, number];
+  type: 'numeric';
+}
+
+export const isNumericChartData = (arg: any): arg is NumericChartData => {
+  return (
+    typeof arg === 'object' &&
+    arg.hasOwnProperty('data') &&
+    arg.hasOwnProperty('id') &&
+    arg.hasOwnProperty('interval') &&
+    arg.hasOwnProperty('stats') &&
+    arg.hasOwnProperty('type') &&
+    arg.type === 'numeric'
+  );
+};
+
+export interface OrdinalDataItem {
+  key: string;
+  key_as_string?: string;
+  doc_count: number;
+}
+
+export interface OrdinalChartData {
+  cardinality: number;
+  data: OrdinalDataItem[];
+  id: string;
+  type: 'ordinal' | 'boolean';
+}
+
+export const isOrdinalChartData = (arg: any): arg is OrdinalChartData => {
+  return (
+    typeof arg === 'object' &&
+    arg.hasOwnProperty('data') &&
+    arg.hasOwnProperty('cardinality') &&
+    arg.hasOwnProperty('id') &&
+    arg.hasOwnProperty('type') &&
+    (arg.type === 'ordinal' || arg.type === 'boolean')
+  );
+};
+
+export interface UnsupportedChartData {
+  id: string;
+  type: 'unsupported';
+}
+
+export const isUnsupportedChartData = (arg: any): arg is UnsupportedChartData => {
+  return typeof arg === 'object' && arg.hasOwnProperty('type') && arg.type === 'unsupported';
+};
+
+export type ChartDataItem = NumericDataItem | OrdinalDataItem;
+export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData;

x-pack/plugins/ml/public/application/components/data_grid/column_chart.tsx

@@ -13,7 +13,9 @@ import { EuiDataGridColumn } from '@elastic/eui';
 
 import './column_chart.scss';
 
-import { isUnsupportedChartData, useColumnChart, ChartData } from './use_column_chart';
+import { isUnsupportedChartData, ChartData } from '../../../../common/types/field_histograms';
+
+import { useColumnChart } from './use_column_chart';
 
 interface Props {
   chartData: ChartData;

x-pack/plugins/ml/public/application/components/data_grid/index.ts

@@ -16,7 +16,7 @@ export {
   useRenderCellValue,
   getProcessedFields,
 } from './common';
-export { getFieldType, ChartData } from './use_column_chart';
+export { getFieldType } from './use_column_chart';
 export { useDataGrid } from './use_data_grid';
 export { DataGrid } from './data_grid';
 export {

x-pack/plugins/ml/public/application/components/data_grid/types.ts

@@ -11,10 +11,10 @@ import { EuiDataGridPaginationProps, EuiDataGridSorting, EuiDataGridColumn } fro
 
 import { Dictionary } from '../../../../common/types/common';
 import { HitsTotalRelation } from '../../../../common/types/es_client';
+import { ChartData } from '../../../../common/types/field_histograms';
 
 import { INDEX_STATUS } from '../../data_frame_analytics/common/analytics';
 
-import { ChartData } from './use_column_chart';
 import { FeatureImportanceBaseline } from '../../../../common/types/feature_importance';
 
 export type ColumnId = string;

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.tsx

@@ -13,17 +13,15 @@ import '@testing-library/jest-dom/extend-expect';
 import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 
 import {
-  getFieldType,
-  getLegendText,
-  getXScaleType,
   isNumericChartData,
   isOrdinalChartData,
   isUnsupportedChartData,
-  useColumnChart,
   NumericChartData,
   OrdinalChartData,
   UnsupportedChartData,
-} from './use_column_chart';
+} from '../../../../common/types/field_histograms';
+
+import { getFieldType, getLegendText, getXScaleType, useColumnChart } from './use_column_chart';
 
 describe('getFieldType()', () => {
   it('should return the Kibana field type for a given EUI data grid schema', () => {

x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx

@@ -17,6 +17,15 @@ import { i18n } from '@kbn/i18n';
 
 import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 
+import {
+  isNumericChartData,
+  isOrdinalChartData,
+  ChartData,
+  ChartDataItem,
+  NumericDataItem,
+  OrdinalDataItem,
+} from '../../../../common/types/field_histograms';
+
 import { NON_AGGREGATABLE } from './common';
 
 export const hoveredRow$ = new BehaviorSubject<any | null>(null);
@@ -66,68 +75,6 @@ export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYP
   return fieldType;
 };
 
-interface NumericDataItem {
-  key: number;
-  key_as_string?: string | number;
-  doc_count: number;
-}
-
-export interface NumericChartData {
-  data: NumericDataItem[];
-  id: string;
-  interval: number;
-  stats: [number, number];
-  type: 'numeric';
-}
-
-export const isNumericChartData = (arg: any): arg is NumericChartData => {
-  return (
-    typeof arg === 'object' &&
-    arg.hasOwnProperty('data') &&
-    arg.hasOwnProperty('id') &&
-    arg.hasOwnProperty('interval') &&
-    arg.hasOwnProperty('stats') &&
-    arg.hasOwnProperty('type') &&
-    arg.type === 'numeric'
-  );
-};
-
-export interface OrdinalDataItem {
-  key: string;
-  key_as_string?: string;
-  doc_count: number;
-}
-
-export interface OrdinalChartData {
-  cardinality: number;
-  data: OrdinalDataItem[];
-  id: string;
-  type: 'ordinal' | 'boolean';
-}
-
-export const isOrdinalChartData = (arg: any): arg is OrdinalChartData => {
-  return (
-    typeof arg === 'object' &&
-    arg.hasOwnProperty('data') &&
-    arg.hasOwnProperty('cardinality') &&
-    arg.hasOwnProperty('id') &&
-    arg.hasOwnProperty('type') &&
-    (arg.type === 'ordinal' || arg.type === 'boolean')
-  );
-};
-
-export interface UnsupportedChartData {
-  id: string;
-  type: 'unsupported';
-}
-
-export const isUnsupportedChartData = (arg: any): arg is UnsupportedChartData => {
-  return typeof arg === 'object' && arg.hasOwnProperty('type') && arg.type === 'unsupported';
-};
-
-export type ChartDataItem = NumericDataItem | OrdinalDataItem;
-export type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData;
-
 type LegendText = string | JSX.Element;
 export const getLegendText = (
   chartData: ChartData,

x-pack/plugins/ml/public/application/components/data_grid/use_data_grid.tsx

@@ -10,6 +10,7 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import { EuiDataGridSorting, EuiDataGridColumn } from '@elastic/eui';
 
 import { HITS_TOTAL_RELATION } from '../../../../common/types/es_client';
+import { ChartData } from '../../../../common/types/field_histograms';
 
 import { INDEX_STATUS } from '../../data_frame_analytics/common';
 
@@ -26,7 +27,6 @@ import {
   RowCountRelation,
   UseDataGridReturnType,
 } from './types';
-import { ChartData } from './use_column_chart';
 
 export const useDataGrid = (
   columns: EuiDataGridColumn[],

x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_row/boolean_content_preview.tsx

@@ -7,10 +7,10 @@
 
 import React, { FC, useMemo } from 'react';
 import { EuiDataGridColumn } from '@elastic/eui';
+import { OrdinalChartData } from '../../../../../../common/types/field_histograms';
+import { ColumnChart } from '../../../../components/data_grid/column_chart';
 import { FieldDataRowProps } from '../../types';
 import { getTFPercentage } from '../../utils';
-import { ColumnChart } from '../../../../components/data_grid/column_chart';
-import { OrdinalChartData } from '../../../../components/data_grid/use_column_chart';
 
 export const BooleanContentPreview: FC<FieldDataRowProps> = ({ config }) => {
   const chartData = useMemo(() => {

x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_row/top_values_preview.tsx

@@ -7,10 +7,9 @@
 
 import React, { FC } from 'react';
 import { EuiDataGridColumn } from '@elastic/eui';
-import type { FieldDataRowProps } from '../../types/field_data_row';
+import { ChartData, OrdinalDataItem } from '../../../../../../common/types/field_histograms';
 import { ColumnChart } from '../../../../components/data_grid/column_chart';
-import { ChartData } from '../../../../components/data_grid';
-import { OrdinalDataItem } from '../../../../components/data_grid/use_column_chart';
+import type { FieldDataRowProps } from '../../types/field_data_row';
 
 export const TopValuesPreview: FC<FieldDataRowProps> = ({ config }) => {
   const { stats } = config;

x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts

@@ -16,7 +16,7 @@ import {
   buildSamplerAggregation,
   getSamplerAggregationsResponsePath,
 } from '../../lib/query_utils';
-import { AggCardinality } from '../../../common/types/fields';
+import { AggCardinality, RuntimeMappings } from '../../../common/types/fields';
 import { getDatafeedAggregations } from '../../../common/util/datafeed_utils';
 import { Datafeed } from '../../../common/types/anomaly_detection_jobs';
 import { isPopulatedObject } from '../../../common/util/object_utils';
@@ -183,7 +183,8 @@ const getAggIntervals = async (
   indexPatternTitle: string,
   query: any,
   fields: HistogramField[],
-  samplerShardSize: number
+  samplerShardSize: number,
+  runtimeMappings?: RuntimeMappings
 ): Promise<NumericColumnStatsMap> => {
   const numericColumns = fields.filter((field) => {
     return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE;
@@ -210,6 +211,7 @@ const getAggIntervals = async (
       query,
       aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize),
       size: 0,
+      ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}),
     },
   });
 
@@ -240,15 +242,17 @@ export const getHistogramsForFields = async (
   indexPatternTitle: string,
   query: any,
   fields: HistogramField[],
-  samplerShardSize: number
+  samplerShardSize: number,
+  runtimeMappings?: RuntimeMappings
 ) => {
   const { asCurrentUser } = client;
   const aggIntervals = await getAggIntervals(
     client,
     indexPatternTitle,
     query,
     fields,
-    samplerShardSize
+    samplerShardSize,
+    runtimeMappings
   );
 
   const chartDataAggs = fields.reduce((aggs, field) => {
@@ -293,6 +297,7 @@ export const getHistogramsForFields = async (
       query,
       aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize),
       size: 0,
+      ...(runtimeMappings !== undefined ? { runtime_mappings: runtimeMappings } : {}),
     },
   });
 
@@ -607,7 +612,7 @@ export class DataVisualizer {
     // Value count aggregation faster way of checking if field exists than using
     // filter aggregation with exists query.
     const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {};
-    const runtimeMappings: any = {};
+    const runtimeMappings: { runtime_mappings?: RuntimeMappings } = {};
 
     aggregatableFields.forEach((field, i) => {
       const safeFieldName = getSafeAggregationName(field, i);

x-pack/plugins/transform/common/api_schemas/common.ts

@@ -53,3 +53,27 @@ export interface ResponseStatus {
 export interface CommonResponseStatusSchema {
   [key: string]: ResponseStatus;
 }
+
+export const runtimeMappingsSchema = schema.maybe(
+  schema.recordOf(
+    schema.string(),
+    schema.object({
+      type: schema.oneOf([
+        schema.literal('keyword'),
+        schema.literal('long'),
+        schema.literal('double'),
+        schema.literal('date'),
+        schema.literal('ip'),
+        schema.literal('boolean'),
+      ]),
+      script: schema.maybe(
+        schema.oneOf([
+          schema.string(),
+          schema.object({
+            source: schema.string(),
+          }),
+        ])
+      ),
+    })
+  )
+);

x-pack/plugins/transform/common/api_schemas/field_histograms.ts

@@ -7,14 +7,20 @@
 
 import { schema, TypeOf } from '@kbn/config-schema';
 
+import { ChartData } from '../shared_imports';
+
+import { runtimeMappingsSchema } from './common';
+
 export const fieldHistogramsRequestSchema = schema.object({
   /** Query to match documents in the index. */
   query: schema.any(),
   /** The fields to return histogram data. */
   fields: schema.arrayOf(schema.any()),
+  /** Optional runtime mappings */
+  runtimeMappings: runtimeMappingsSchema,
   /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */
   samplerShardSize: schema.number(),
 });
 
 export type FieldHistogramsRequestSchema = TypeOf<typeof fieldHistogramsRequestSchema>;
-export type FieldHistogramsResponseSchema = any[];
+export type FieldHistogramsResponseSchema = ChartData[];