diff --git a/x-pack/plugins/ml/common/constants/field_histograms.ts b/x-pack/plugins/ml/common/constants/field_histograms.ts new file mode 100644 index 0000000000000..5c86c00ac666f --- /dev/null +++ b/x-pack/plugins/ml/common/constants/field_histograms.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// Default sampler shard size used for field histograms +export const DEFAULT_SAMPLER_SHARD_SIZE = 5000; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx index 9af7a869e0e56..d4be2eab13d26 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/data_grid.tsx @@ -20,10 +20,13 @@ import { EuiFlexItem, EuiSpacer, EuiTitle, + EuiToolTip, } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; +import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../common/constants/field_histograms'; + import { INDEX_STATUS } from '../../data_frame_analytics/common'; import { euiDataGridStyle, euiDataGridToolbarSettings } from './common'; @@ -193,21 +196,31 @@ export const DataGrid: FC = memo( ...(chartsButtonVisible ? { additionalControls: ( - - {i18n.translate('xpack.ml.dataGrid.histogramButtonText', { - defaultMessage: 'Histogram charts', + + > + + {i18n.translate('xpack.ml.dataGrid.histogramButtonText', { + defaultMessage: 'Histogram charts', + })} + + ), } : {}), diff --git a/x-pack/plugins/ml/public/application/components/data_grid/index.ts b/x-pack/plugins/ml/public/application/components/data_grid/index.ts index 80bc6b861f742..4bbd3595e5a7e 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/index.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/index.ts @@ -12,7 +12,7 @@ export { showDataGridColumnChartErrorMessageToast, useRenderCellValue, } from './common'; -export { fetchChartsData, ChartData } from './use_column_chart'; +export { getFieldType, ChartData } from './use_column_chart'; export { useDataGrid } from './use_data_grid'; export { DataGrid } from './data_grid'; export { diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts new file mode 100644 index 0000000000000..1b35ef238d09e --- /dev/null +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.test.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getFieldType } from './use_column_chart'; + +describe('getFieldType()', () => { + it('should return the Kibana field type for a given EUI data grid schema', () => { + expect(getFieldType('text')).toBe('string'); + expect(getFieldType('datetime')).toBe('date'); + expect(getFieldType('numeric')).toBe('number'); + expect(getFieldType('boolean')).toBe('boolean'); + expect(getFieldType('json')).toBe('object'); + expect(getFieldType('non-aggregatable')).toBe(undefined); + }); +}); diff --git a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx index 6b207a999eb52..a762c44e243bf 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx +++ b/x-pack/plugins/ml/public/application/components/data_grid/use_column_chart.tsx @@ -16,8 +16,6 @@ import { i18n } from '@kbn/i18n'; import { KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; -import { stringHash } from '../../../../common/util/string_utils'; - import { NON_AGGREGATABLE } from './common'; export const hoveredRow$ = new BehaviorSubject(null); @@ -40,7 +38,7 @@ const getXScaleType = (kbnFieldType: KBN_FIELD_TYPES | undefined): XScaleType => } }; -const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => { +export const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | undefined => { if (schema === NON_AGGREGATABLE) { return undefined; } @@ -67,188 +65,6 @@ const getFieldType = (schema: EuiDataGridColumn['schema']): KBN_FIELD_TYPES | un return fieldType; }; -interface NumericColumnStats { - interval: number; - min: number; - max: number; -} -type NumericColumnStatsMap = Record; -const getAggIntervals = async ( - indexPatternTitle: string, - esSearch: (payload: any) => Promise, - query: any, - columnTypes: EuiDataGridColumn[] -): Promise => { - const numericColumns = columnTypes.filter((cT) => { - const fieldType = getFieldType(cT.schema); - return fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE; - }); - - if (numericColumns.length === 0) { - return {}; - } - - const minMaxAggs = numericColumns.reduce((aggs, c) => { - const id = stringHash(c.id); - aggs[id] = { - stats: { - field: c.id, - }, - }; - return aggs; - }, {} as Record); - - const respStats = await esSearch({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: minMaxAggs, - size: 0, - }, - }); - - return Object.keys(respStats.aggregations).reduce((p, aggName) => { - const stats = [respStats.aggregations[aggName].min, respStats.aggregations[aggName].max]; - if (!stats.includes(null)) { - const delta = respStats.aggregations[aggName].max - respStats.aggregations[aggName].min; - - let aggInterval = 1; - - if (delta > MAX_CHART_COLUMNS) { - aggInterval = Math.round(delta / MAX_CHART_COLUMNS); - } - - if (delta <= 1) { - aggInterval = delta / MAX_CHART_COLUMNS; - } - - p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; - } - - return p; - }, {} as NumericColumnStatsMap); -}; - -interface AggHistogram { - histogram: { - field: string; - interval: number; - }; -} - -interface AggCardinality { - cardinality: { - field: string; - }; -} - -interface AggTerms { - terms: { - field: string; - size: number; - }; -} - -type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; - -export const fetchChartsData = async ( - indexPatternTitle: string, - esSearch: (payload: any) => Promise, - query: any, - columnTypes: EuiDataGridColumn[] -): Promise => { - const aggIntervals = await getAggIntervals(indexPatternTitle, esSearch, query, columnTypes); - - const chartDataAggs = columnTypes.reduce((aggs, c) => { - const fieldType = getFieldType(c.schema); - const id = stringHash(c.id); - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] !== undefined) { - aggs[`${id}_histogram`] = { - histogram: { - field: c.id, - interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, - }, - }; - } - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - if (fieldType === KBN_FIELD_TYPES.STRING) { - aggs[`${id}_cardinality`] = { - cardinality: { - field: c.id, - }, - }; - } - aggs[`${id}_terms`] = { - terms: { - field: c.id, - size: MAX_CHART_COLUMNS, - }, - }; - } - return aggs; - }, {} as Record); - - if (Object.keys(chartDataAggs).length === 0) { - return []; - } - - const respChartsData = await esSearch({ - index: indexPatternTitle, - size: 0, - body: { - query, - aggs: chartDataAggs, - size: 0, - }, - }); - - const chartsData: ChartData[] = columnTypes.map( - (c): ChartData => { - const fieldType = getFieldType(c.schema); - const id = stringHash(c.id); - - if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { - if (aggIntervals[id] === undefined) { - return { - type: 'numeric', - data: [], - interval: 0, - stats: [0, 0], - id: c.id, - }; - } - - return { - data: respChartsData.aggregations[`${id}_histogram`].buckets, - interval: aggIntervals[id].interval, - stats: [aggIntervals[id].min, aggIntervals[id].max], - type: 'numeric', - id: c.id, - }; - } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { - return { - type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', - cardinality: - fieldType === KBN_FIELD_TYPES.STRING - ? respChartsData.aggregations[`${id}_cardinality`].value - : 2, - data: respChartsData.aggregations[`${id}_terms`].buckets, - id: c.id, - }; - } - - return { - type: 'unsupported', - id: c.id, - }; - } - ); - - return chartsData; -}; - interface NumericDataItem { key: number; key_as_string?: string; diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index ee0e5c1955ead..2cecffc993257 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -4,15 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; import { CoreSetup } from 'src/core/public'; import { IndexPattern } from '../../../../../../../../../src/plugins/data/public'; + +import { DataLoader } from '../../../../datavisualizer/index_based/data_loader'; + import { - fetchChartsData, + getFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, showDataGridColumnChartErrorMessageToast, @@ -103,13 +106,20 @@ export const useIndexData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [indexPattern.title, JSON.stringify([query, pagination, sortingColumns])]); + const dataLoader = useMemo(() => new DataLoader(indexPattern, toastNotifications), [ + indexPattern, + ]); + const fetchColumnChartsData = async function () { try { - const columnChartsData = await fetchChartsData( - indexPattern.title, - ml.esSearch, - query, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + const columnChartsData = await dataLoader.loadFieldHistograms( + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })), + query ); dataGrid.setColumnCharts(columnChartsData); } catch (e) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts index 796670f6a864d..98dd40986e32b 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/exploration_results_table/use_exploration_results.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; @@ -12,16 +12,17 @@ import { CoreSetup } from 'src/core/public'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; + import { - fetchChartsData, getDataGridSchemasFromFieldTypes, + getFieldType, showDataGridColumnChartErrorMessageToast, useDataGrid, useRenderCellValue, UseIndexDataReturnType, } from '../../../../../components/data_grid'; import { SavedSearchQuery } from '../../../../../contexts/ml'; -import { ml } from '../../../../../services/ml_api_service'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; import { @@ -72,14 +73,23 @@ export const useExplorationResults = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [jobConfig && jobConfig.id, dataGrid.pagination, searchQuery, dataGrid.sortingColumns]); + const dataLoader = useMemo( + () => + indexPattern !== undefined ? new DataLoader(indexPattern, toastNotifications) : undefined, + [indexPattern] + ); + const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined) { - const columnChartsData = await fetchChartsData( - jobConfig.dest.index, - ml.esSearch, - searchQuery, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + if (jobConfig !== undefined && dataLoader !== undefined) { + const columnChartsData = await dataLoader.loadFieldHistograms( + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })), + searchQuery ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts index beb6836bf801f..90294a09c0adc 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_exploration/components/outlier_exploration/use_outlier_data.ts @@ -4,19 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import { useEffect } from 'react'; +import { useEffect, useMemo } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; import { IndexPattern } from '../../../../../../../../../../src/plugins/data/public'; +import { DataLoader } from '../../../../../datavisualizer/index_based/data_loader'; + import { useColorRange, COLOR_RANGE, COLOR_RANGE_SCALE, } from '../../../../../components/color_range_legend'; import { - fetchChartsData, + getFieldType, getDataGridSchemasFromFieldTypes, showDataGridColumnChartErrorMessageToast, useDataGrid, @@ -24,7 +26,6 @@ import { UseIndexDataReturnType, } from '../../../../../components/data_grid'; import { SavedSearchQuery } from '../../../../../contexts/ml'; -import { ml } from '../../../../../services/ml_api_service'; import { getToastNotifications } from '../../../../../util/dependency_cache'; import { getIndexData, getIndexFields, DataFrameAnalyticsConfig } from '../../../../common'; @@ -79,14 +80,25 @@ export const useOutlierData = ( // eslint-disable-next-line react-hooks/exhaustive-deps }, [jobConfig && jobConfig.id, dataGrid.pagination, searchQuery, dataGrid.sortingColumns]); + const dataLoader = useMemo( + () => + indexPattern !== undefined + ? new DataLoader(indexPattern, getToastNotifications()) + : undefined, + [indexPattern] + ); + const fetchColumnChartsData = async function () { try { - if (jobConfig !== undefined) { - const columnChartsData = await fetchChartsData( - jobConfig.dest.index, - ml.esSearch, - searchQuery, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + if (jobConfig !== undefined && dataLoader !== undefined) { + const columnChartsData = await dataLoader.loadFieldHistograms( + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })), + searchQuery ); dataGrid.setColumnCharts(columnChartsData); } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts index 5618f701e4c5f..50278c300d103 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/index.ts @@ -5,4 +5,4 @@ */ export { FieldVisConfig } from './field_vis_config'; -export { FieldRequestConfig } from './request'; +export { FieldHistogramRequestConfig, FieldRequestConfig } from './request'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts index 9a886cbc899c2..fd4888b8729c1 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/common/request.ts @@ -4,6 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ +import { KBN_FIELD_TYPES } from '../../../../../../../../src/plugins/data/public'; + import { ML_JOB_FIELD_TYPES } from '../../../../../common/constants/field_types'; export interface FieldRequestConfig { @@ -11,3 +13,8 @@ export interface FieldRequestConfig { type: ML_JOB_FIELD_TYPES; cardinality: number; } + +export interface FieldHistogramRequestConfig { + fieldName: string; + type?: KBN_FIELD_TYPES; +} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts index a08821c65bfe7..34f86ffa18788 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/data_loader/data_loader.ts @@ -6,15 +6,17 @@ import { i18n } from '@kbn/i18n'; -import { getToastNotifications } from '../../../util/dependency_cache'; +import { CoreSetup } from 'src/core/public'; + import { IndexPattern } from '../../../../../../../../src/plugins/data/public'; import { SavedSearchQuery } from '../../../contexts/ml'; import { OMIT_FIELDS } from '../../../../../common/constants/field_types'; import { IndexPatternTitle } from '../../../../../common/types/kibana'; +import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../common/constants/field_histograms'; import { ml } from '../../../services/ml_api_service'; -import { FieldRequestConfig } from '../common'; +import { FieldHistogramRequestConfig, FieldRequestConfig } from '../common'; // Maximum number of examples to obtain for text type fields. const MAX_EXAMPLES_DEFAULT: number = 10; @@ -23,10 +25,15 @@ export class DataLoader { private _indexPattern: IndexPattern; private _indexPatternTitle: IndexPatternTitle = ''; private _maxExamples: number = MAX_EXAMPLES_DEFAULT; + private _toastNotifications: CoreSetup['notifications']['toasts']; - constructor(indexPattern: IndexPattern, kibanaConfig: any) { + constructor( + indexPattern: IndexPattern, + toastNotifications: CoreSetup['notifications']['toasts'] + ) { this._indexPattern = indexPattern; this._indexPatternTitle = indexPattern.title; + this._toastNotifications = toastNotifications; } async loadOverallData( @@ -90,10 +97,24 @@ export class DataLoader { return stats; } + async loadFieldHistograms( + fields: FieldHistogramRequestConfig[], + query: string | SavedSearchQuery, + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE + ): Promise { + const stats = await ml.getVisualizerFieldHistograms({ + indexPatternTitle: this._indexPatternTitle, + query, + fields, + samplerShardSize, + }); + + return stats; + } + displayError(err: any) { - const toastNotifications = getToastNotifications(); if (err.statusCode === 500) { - toastNotifications.addDanger( + this._toastNotifications.addDanger( i18n.translate('xpack.ml.datavisualizer.dataLoader.internalServerErrorMessage', { defaultMessage: 'Error loading data in index {index}. {message}. ' + @@ -105,7 +126,7 @@ export class DataLoader { }) ); } else { - toastNotifications.addDanger( + this._toastNotifications.addDanger( i18n.translate('xpack.ml.datavisualizer.page.errorLoadingDataMessage', { defaultMessage: 'Error loading data in index {index}. {message}', values: { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx index 97b4043c9fd64..3c332d305d7e9 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/page.tsx @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, Fragment, useEffect, useState } from 'react'; +import React, { FC, Fragment, useEffect, useMemo, useState } from 'react'; import { merge } from 'rxjs'; import { i18n } from '@kbn/i18n'; @@ -43,6 +43,7 @@ import { kbnTypeToMLJobType } from '../../util/field_types_utils'; import { useTimefilter } from '../../contexts/kibana'; import { timeBasedIndexCheck, getQueryFromSavedSearch } from '../../util/index_utils'; import { getTimeBucketsFromCache } from '../../util/time_buckets'; +import { getToastNotifications } from '../../util/dependency_cache'; import { useUrlState } from '../../util/url_state'; import { FieldRequestConfig, FieldVisConfig } from './common'; import { ActionsPanel } from './components/actions_panel'; @@ -107,7 +108,10 @@ export const Page: FC = () => { autoRefreshSelector: true, }); - const dataLoader = new DataLoader(currentIndexPattern, kibanaConfig); + const dataLoader = useMemo(() => new DataLoader(currentIndexPattern, getToastNotifications()), [ + currentIndexPattern, + ]); + const [globalState, setGlobalState] = useUrlState('_g'); useEffect(() => { if (globalState?.time !== undefined) { diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index d1b6f95f32bed..599e4d4bb8a10 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -27,7 +27,10 @@ import { ModelSnapshot, } from '../../../../common/types/anomaly_detection_jobs'; import { ES_AGGREGATION } from '../../../../common/constants/aggregation_types'; -import { FieldRequestConfig } from '../../datavisualizer/index_based/common'; +import { + FieldHistogramRequestConfig, + FieldRequestConfig, +} from '../../datavisualizer/index_based/common'; import { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; import { getHttp } from '../../util/dependency_cache'; @@ -494,6 +497,30 @@ export function mlApiServicesProvider(httpService: HttpService) { }); }, + getVisualizerFieldHistograms({ + indexPatternTitle, + query, + fields, + samplerShardSize, + }: { + indexPatternTitle: string; + query: any; + fields: FieldHistogramRequestConfig[]; + samplerShardSize?: number; + }) { + const body = JSON.stringify({ + query, + fields, + samplerShardSize, + }); + + return httpService.http({ + path: `${basePath()}/data_visualizer/get_field_histograms/${indexPatternTitle}`, + method: 'POST', + body, + }); + }, + getVisualizerOverallStats({ indexPatternTitle, query, diff --git a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts index d58c797b446db..d1a4a0b585fbb 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/data_visualizer.ts @@ -4,10 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { LegacyCallAPIOptions, LegacyAPICaller } from 'kibana/server'; +import { LegacyAPICaller } from 'kibana/server'; import _ from 'lodash'; +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/server'; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types'; import { getSafeAggregationName } from '../../../common/util/job_utils'; +import { stringHash } from '../../../common/util/string_utils'; import { buildBaseFilterCriteria, buildSamplerAggregation, @@ -19,6 +21,8 @@ const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000; const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200; const FIELDS_REQUEST_BATCH_SIZE = 10; +const MAX_CHART_COLUMNS = 20; + interface FieldData { fieldName: string; existsInDocs: boolean; @@ -35,6 +39,11 @@ export interface Field { cardinality: number; } +export interface HistogramField { + fieldName: string; + type: string; +} + interface Distribution { percentiles: any[]; minPercentile: number; @@ -98,6 +107,70 @@ interface FieldExamples { examples: any[]; } +interface NumericColumnStats { + interval: number; + min: number; + max: number; +} +type NumericColumnStatsMap = Record; + +interface AggHistogram { + histogram: { + field: string; + interval: number; + }; +} + +interface AggCardinality { + cardinality: { + field: string; + }; +} + +interface AggTerms { + terms: { + field: string; + size: number; + }; +} + +interface NumericDataItem { + key: number; + key_as_string?: string; + doc_count: number; +} + +interface NumericChartData { + data: NumericDataItem[]; + id: string; + interval: number; + stats: [number, number]; + type: 'numeric'; +} + +interface OrdinalDataItem { + key: string; + key_as_string?: string; + doc_count: number; +} + +interface OrdinalChartData { + type: 'ordinal' | 'boolean'; + cardinality: number; + data: OrdinalDataItem[]; + id: string; +} + +interface UnsupportedChartData { + id: string; + type: 'unsupported'; +} + +type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms; + +// type ChartDataItem = NumericDataItem | OrdinalDataItem; +type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData; + type BatchStats = | NumericFieldStats | StringFieldStats @@ -106,12 +179,176 @@ type BatchStats = | DocumentCountStats | FieldExamples; +const getAggIntervals = async ( + callAsCurrentUser: LegacyAPICaller, + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number +): Promise => { + const numericColumns = fields.filter((field) => { + return field.type === KBN_FIELD_TYPES.NUMBER || field.type === KBN_FIELD_TYPES.DATE; + }); + + if (numericColumns.length === 0) { + return {}; + } + + const minMaxAggs = numericColumns.reduce((aggs, c) => { + const id = stringHash(c.fieldName); + aggs[id] = { + stats: { + field: c.fieldName, + }, + }; + return aggs; + }, {} as Record); + + const respStats = await callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: buildSamplerAggregation(minMaxAggs, samplerShardSize), + size: 0, + }, + }); + + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = + aggsPath.length > 0 ? _.get(respStats.aggregations, aggsPath) : respStats.aggregations; + + return Object.keys(aggregations).reduce((p, aggName) => { + const stats = [aggregations[aggName].min, aggregations[aggName].max]; + if (!stats.includes(null)) { + const delta = aggregations[aggName].max - aggregations[aggName].min; + + let aggInterval = 1; + + if (delta > MAX_CHART_COLUMNS || delta <= 1) { + aggInterval = delta / (MAX_CHART_COLUMNS - 1); + } + + p[aggName] = { interval: aggInterval, min: stats[0], max: stats[1] }; + } + + return p; + }, {} as NumericColumnStatsMap); +}; + +// export for re-use by transforms plugin +export const getHistogramsForFields = async ( + callAsCurrentUser: LegacyAPICaller, + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number +) => { + const aggIntervals = await getAggIntervals( + callAsCurrentUser, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + const chartDataAggs = fields.reduce((aggs, field) => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(fieldName); + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { + if (aggIntervals[id] !== undefined) { + aggs[`${id}_histogram`] = { + histogram: { + field: fieldName, + interval: aggIntervals[id].interval !== 0 ? aggIntervals[id].interval : 1, + }, + }; + } + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { + if (fieldType === KBN_FIELD_TYPES.STRING) { + aggs[`${id}_cardinality`] = { + cardinality: { + field: fieldName, + }, + }; + } + aggs[`${id}_terms`] = { + terms: { + field: fieldName, + size: MAX_CHART_COLUMNS, + }, + }; + } + return aggs; + }, {} as Record); + + if (Object.keys(chartDataAggs).length === 0) { + return []; + } + + const respChartsData = await callAsCurrentUser('search', { + index: indexPatternTitle, + size: 0, + body: { + query, + aggs: buildSamplerAggregation(chartDataAggs, samplerShardSize), + size: 0, + }, + }); + + const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize); + const aggregations = + aggsPath.length > 0 + ? _.get(respChartsData.aggregations, aggsPath) + : respChartsData.aggregations; + + const chartsData: ChartData[] = fields.map( + (field): ChartData => { + const fieldName = field.fieldName; + const fieldType = field.type; + const id = stringHash(field.fieldName); + + if (fieldType === KBN_FIELD_TYPES.NUMBER || fieldType === KBN_FIELD_TYPES.DATE) { + if (aggIntervals[id] === undefined) { + return { + type: 'numeric', + data: [], + interval: 0, + stats: [0, 0], + id: fieldName, + }; + } + + return { + data: aggregations[`${id}_histogram`].buckets, + interval: aggIntervals[id].interval, + stats: [aggIntervals[id].min, aggIntervals[id].max], + type: 'numeric', + id: fieldName, + }; + } else if (fieldType === KBN_FIELD_TYPES.STRING || fieldType === KBN_FIELD_TYPES.BOOLEAN) { + return { + type: fieldType === KBN_FIELD_TYPES.STRING ? 'ordinal' : 'boolean', + cardinality: + fieldType === KBN_FIELD_TYPES.STRING ? aggregations[`${id}_cardinality`].value : 2, + data: aggregations[`${id}_terms`].buckets, + id: fieldName, + }; + } + + return { + type: 'unsupported', + id: fieldName, + }; + } + ); + + return chartsData; +}; + export class DataVisualizer { - callAsCurrentUser: ( - endpoint: string, - clientParams: Record, - options?: LegacyCallAPIOptions - ) => Promise; + callAsCurrentUser: LegacyAPICaller; constructor(callAsCurrentUser: LegacyAPICaller) { this.callAsCurrentUser = callAsCurrentUser; @@ -200,6 +437,24 @@ export class DataVisualizer { return stats; } + // Obtains binned histograms for supplied list of fields. The statistics for each field in the + // returned array depend on the type of the field (keyword, number, date etc). + // Sampling will be used if supplied samplerShardSize > 0. + async getHistogramsForFields( + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number + ): Promise { + return await getHistogramsForFields( + this.callAsCurrentUser, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + } + // Obtains statistics for supplied list of fields. The statistics for each field in the // returned array depend on the type of the field (keyword, number, date etc). // Sampling will be used if supplied samplerShardSize > 0. diff --git a/x-pack/plugins/ml/server/models/data_visualizer/index.ts b/x-pack/plugins/ml/server/models/data_visualizer/index.ts index ed44e9b12e1d1..ca1df0fe8300c 100644 --- a/x-pack/plugins/ml/server/models/data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/data_visualizer/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { DataVisualizer } from './data_visualizer'; +export { getHistogramsForFields, DataVisualizer } from './data_visualizer'; diff --git a/x-pack/plugins/ml/server/routes/data_visualizer.ts b/x-pack/plugins/ml/server/routes/data_visualizer.ts index 04008a896a1a2..9dd010e105b6e 100644 --- a/x-pack/plugins/ml/server/routes/data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/data_visualizer.ts @@ -7,8 +7,9 @@ import { RequestHandlerContext } from 'kibana/server'; import { wrapError } from '../client/error_wrapper'; import { DataVisualizer } from '../models/data_visualizer'; -import { Field } from '../models/data_visualizer/data_visualizer'; +import { Field, HistogramField } from '../models/data_visualizer/data_visualizer'; import { + dataVisualizerFieldHistogramsSchema, dataVisualizerFieldStatsSchema, dataVisualizerOverallStatsSchema, indexPatternTitleSchema, @@ -65,10 +66,68 @@ function getStatsForFields( ); } +function getHistogramsForFields( + context: RequestHandlerContext, + indexPatternTitle: string, + query: any, + fields: HistogramField[], + samplerShardSize: number +) { + const dv = new DataVisualizer(context.ml!.mlClient.callAsCurrentUser); + return dv.getHistogramsForFields(indexPatternTitle, query, fields, samplerShardSize); +} + /** * Routes for the index data visualizer. */ export function dataVisualizerRoutes({ router, mlLicense }: RouteInitialization) { + /** + * @apiGroup DataVisualizer + * + * @api {post} /api/ml/data_visualizer/get_field_stats/:indexPatternTitle Get histograms for fields + * @apiName GetHistogramsForFields + * @apiDescription Returns the histograms on a list fields in the specified index pattern. + * + * @apiSchema (params) indexPatternTitleSchema + * @apiSchema (body) dataVisualizerFieldHistogramsSchema + * + * @apiSuccess {Object} fieldName histograms by field, keyed on the name of the field. + */ + router.post( + { + path: '/api/ml/data_visualizer/get_field_histograms/{indexPatternTitle}', + validate: { + params: indexPatternTitleSchema, + body: dataVisualizerFieldHistogramsSchema, + }, + options: { + tags: ['access:ml:canAccessML'], + }, + }, + mlLicense.basicLicenseAPIGuard(async (context, request, response) => { + try { + const { + params: { indexPatternTitle }, + body: { query, fields, samplerShardSize }, + } = request; + + const results = await getHistogramsForFields( + context, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + return response.ok({ + body: results, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup DataVisualizer * diff --git a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts index b2d665954bd4d..24e45514e1efc 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_visualizer_schema.ts @@ -11,6 +11,15 @@ export const indexPatternTitleSchema = schema.object({ indexPatternTitle: schema.string(), }); +export const dataVisualizerFieldHistogramsSchema = schema.object({ + /** Query to match documents in the index. */ + query: schema.any(), + /** The fields to return histogram data. */ + fields: schema.arrayOf(schema.any()), + /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ + samplerShardSize: schema.number(), +}); + export const dataVisualizerFieldStatsSchema = schema.object({ /** Query to match documents in the index. */ query: schema.any(), diff --git a/x-pack/plugins/ml/server/shared.ts b/x-pack/plugins/ml/server/shared.ts index 3fca8ea1ba047..100433b23f7d1 100644 --- a/x-pack/plugins/ml/server/shared.ts +++ b/x-pack/plugins/ml/server/shared.ts @@ -8,3 +8,4 @@ export * from '../common/types/anomalies'; export * from '../common/types/anomaly_detection_jobs'; export * from './lib/capabilities/errors'; export { ModuleSetupPayload } from './shared_services/providers/modules'; +export { getHistogramsForFields } from './models/data_visualizer/'; diff --git a/x-pack/plugins/transform/public/app/hooks/use_api.ts b/x-pack/plugins/transform/public/app/hooks/use_api.ts index 56528370a3ab9..1d2752b9e939d 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_api.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_api.ts @@ -5,6 +5,9 @@ */ import { useMemo } from 'react'; + +import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/public'; + import { TransformId, TransformEndpointRequest, @@ -17,6 +20,15 @@ import { useAppDependencies } from '../app_dependencies'; import { GetTransformsResponse, PreviewRequestBody } from '../common'; import { EsIndex } from './use_api_types'; +import { SavedSearchQuery } from './use_search_items'; + +// Default sampler shard size used for field histograms +export const DEFAULT_SAMPLER_SHARD_SIZE = 5000; + +export interface FieldHistogramRequestConfig { + fieldName: string; + type?: KBN_FIELD_TYPES; +} export const useApi = () => { const { http } = useAppDependencies(); @@ -85,6 +97,20 @@ export const useApi = () => { getIndices(): Promise { return http.get(`/api/index_management/indices`); }, + getHistogramsForFields( + indexPatternTitle: string, + fields: FieldHistogramRequestConfig[], + query: string | SavedSearchQuery, + samplerShardSize = DEFAULT_SAMPLER_SHARD_SIZE + ) { + return http.post(`${API_BASE_PATH}field_histograms/${indexPatternTitle}`, { + body: JSON.stringify({ + query, + fields, + samplerShardSize, + }), + }); + }, }), [http] ); diff --git a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts index c821c183ad370..ad5850f26be2e 100644 --- a/x-pack/plugins/transform/public/app/hooks/use_index_data.ts +++ b/x-pack/plugins/transform/public/app/hooks/use_index_data.ts @@ -9,7 +9,7 @@ import { useEffect } from 'react'; import { EuiDataGridColumn } from '@elastic/eui'; import { - fetchChartsData, + getFieldType, getDataGridSchemaFromKibanaFieldType, getFieldsFromKibanaIndexPattern, getErrorMessage, @@ -107,13 +107,16 @@ export const useIndexData = ( const fetchColumnChartsData = async function () { try { - const columnChartsData = await fetchChartsData( + const columnChartsData = await api.getHistogramsForFields( indexPattern.title, - api.esSearch, - isDefaultQuery(query) ? matchAllQuery : query, - columns.filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + columns + .filter((cT) => dataGrid.visibleColumns.includes(cT.id)) + .map((cT) => ({ + fieldName: cT.id, + type: getFieldType(cT.schema), + })), + isDefaultQuery(query) ? matchAllQuery : query ); - setColumnCharts(columnChartsData); } catch (e) { showDataGridColumnChartErrorMessageToast(e, toastNotifications); diff --git a/x-pack/plugins/transform/public/shared_imports.ts b/x-pack/plugins/transform/public/shared_imports.ts index e0bbcd0b5d9db..abbc39dd6c728 100644 --- a/x-pack/plugins/transform/public/shared_imports.ts +++ b/x-pack/plugins/transform/public/shared_imports.ts @@ -14,7 +14,7 @@ export { } from '../../../../src/plugins/es_ui_shared/public'; export { - fetchChartsData, + getFieldType, getErrorMessage, extractErrorMessage, formatHumanReadableDateTimeSeconds, diff --git a/x-pack/plugins/transform/server/routes/api/field_histograms.ts b/x-pack/plugins/transform/server/routes/api/field_histograms.ts new file mode 100644 index 0000000000000..d602e49338846 --- /dev/null +++ b/x-pack/plugins/transform/server/routes/api/field_histograms.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { wrapEsError } from '../../../../../legacy/server/lib/create_router/error_wrappers'; + +import { getHistogramsForFields } from '../../shared_imports'; +import { RouteDependencies } from '../../types'; + +import { addBasePath } from '../index'; + +import { wrapError } from './error_utils'; +import { fieldHistogramsSchema, indexPatternTitleSchema, IndexPatternTitleSchema } from './schema'; + +export function registerFieldHistogramsRoutes({ router, license }: RouteDependencies) { + router.post( + { + path: addBasePath('field_histograms/{indexPatternTitle}'), + validate: { + params: indexPatternTitleSchema, + body: fieldHistogramsSchema, + }, + }, + license.guardApiRoute(async (ctx, req, res) => { + const { indexPatternTitle } = req.params as IndexPatternTitleSchema; + const { query, fields, samplerShardSize } = req.body; + + try { + const resp = await getHistogramsForFields( + ctx.transform!.dataClient.callAsCurrentUser, + indexPatternTitle, + query, + fields, + samplerShardSize + ); + + return res.ok({ body: resp }); + } catch (e) { + return res.customError(wrapError(wrapEsError(e))); + } + }) + ); +} diff --git a/x-pack/plugins/transform/server/routes/api/schema.ts b/x-pack/plugins/transform/server/routes/api/schema.ts index 7da3f1ccfe55e..8aadef81b221b 100644 --- a/x-pack/plugins/transform/server/routes/api/schema.ts +++ b/x-pack/plugins/transform/server/routes/api/schema.ts @@ -5,6 +5,24 @@ */ import { schema } from '@kbn/config-schema'; +export const fieldHistogramsSchema = schema.object({ + /** Query to match documents in the index. */ + query: schema.any(), + /** The fields to return histogram data. */ + fields: schema.arrayOf(schema.any()), + /** Number of documents to be collected in the sample processed on each shard, or -1 for no sampling. */ + samplerShardSize: schema.number(), +}); + +export const indexPatternTitleSchema = schema.object({ + /** Title of the index pattern for which to return stats. */ + indexPatternTitle: schema.string(), +}); + +export interface IndexPatternTitleSchema { + indexPatternTitle: string; +} + export const schemaTransformId = { params: schema.object({ transformId: schema.string(), diff --git a/x-pack/plugins/transform/server/routes/index.ts b/x-pack/plugins/transform/server/routes/index.ts index 07c21e58e64e4..4f35b094017a4 100644 --- a/x-pack/plugins/transform/server/routes/index.ts +++ b/x-pack/plugins/transform/server/routes/index.ts @@ -6,6 +6,7 @@ import { RouteDependencies } from '../types'; +import { registerFieldHistogramsRoutes } from './api/field_histograms'; import { registerPrivilegesRoute } from './api/privileges'; import { registerTransformsRoutes } from './api/transforms'; @@ -15,6 +16,7 @@ export const addBasePath = (uri: string): string => `${API_BASE_PATH}${uri}`; export class ApiRoutes { setup(dependencies: RouteDependencies) { + registerFieldHistogramsRoutes(dependencies); registerPrivilegesRoute(dependencies); registerTransformsRoutes(dependencies); } diff --git a/x-pack/plugins/transform/server/shared_imports.ts b/x-pack/plugins/transform/server/shared_imports.ts new file mode 100644 index 0000000000000..d1f86ac375721 --- /dev/null +++ b/x-pack/plugins/transform/server/shared_imports.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { getHistogramsForFields } from '../../ml/server'; diff --git a/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts new file mode 100644 index 0000000000000..8b21c367d29f6 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/data_visualizer/get_field_histograms.ts @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + const fieldHistogramsTestData = { + testTitle: 'returns histogram data for fields', + index: 'ft_farequote', + user: USER.ML_POWERUSER, + requestBody: { + query: { bool: { should: [{ match_phrase: { airline: 'JZA' } }], minimum_should_match: 1 } }, + fields: [ + { fieldName: '@timestamp', type: 'date' }, + { fieldName: 'airline', type: 'string' }, + { fieldName: 'responsetime', type: 'number' }, + ], + samplerShardSize: -1, // No sampling, as otherwise counts could vary on each run. + }, + expected: { + responseCode: 200, + responseBody: [ + { + dataLength: 20, + type: 'numeric', + id: '@timestamp', + }, + { type: 'ordinal', dataLength: 1, id: 'airline' }, + { + dataLength: 20, + type: 'numeric', + id: 'responsetime', + }, + ], + }, + }; + + const errorTestData = { + testTitle: 'returns error for index which does not exist', + index: 'ft_farequote_not_exists', + user: USER.ML_POWERUSER, + requestBody: { + query: { bool: { must: [{ match_all: {} }] } }, + fields: [{ fieldName: 'responsetime', type: 'number' }], + samplerShardSize: -1, + }, + expected: { + responseCode: 404, + responseBody: { + statusCode: 404, + error: 'Not Found', + message: + '[index_not_found_exception] no such index [ft_farequote_not_exists], with { resource.type="index_or_alias" & resource.id="ft_farequote_not_exists" & index_uuid="_na_" & index="ft_farequote_not_exists" }', + }, + }, + }; + + async function runGetFieldHistogramsRequest( + index: string, + user: USER, + requestBody: object, + expectedResponsecode: number + ): Promise { + const { body } = await supertest + .post(`/api/ml/data_visualizer/get_field_histograms/${index}`) + .auth(user, ml.securityCommon.getPasswordForUser(user)) + .set(COMMON_REQUEST_HEADERS) + .send(requestBody) + .expect(expectedResponsecode); + + return body; + } + + describe('get_field_histograms', function () { + before(async () => { + await esArchiver.loadIfNeeded('ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + }); + + it(`${fieldHistogramsTestData.testTitle}`, async () => { + const body = await runGetFieldHistogramsRequest( + fieldHistogramsTestData.index, + fieldHistogramsTestData.user, + fieldHistogramsTestData.requestBody, + fieldHistogramsTestData.expected.responseCode + ); + + const expected = fieldHistogramsTestData.expected; + + const actual = body.map((b: any) => ({ + dataLength: b.data.length, + type: b.type, + id: b.id, + })); + expect(actual).to.eql(expected.responseBody); + }); + + it(`${errorTestData.testTitle}`, async () => { + const body = await runGetFieldHistogramsRequest( + errorTestData.index, + errorTestData.user, + errorTestData.requestBody, + errorTestData.expected.responseCode + ); + + expect(body.error).to.eql(errorTestData.expected.responseBody.error); + expect(body.message).to.eql(errorTestData.expected.responseBody.message); + }); + }); +}; diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index 6cdb9caa1e2db..4ae93296f9be0 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -37,6 +37,18 @@ export default function ({ getService }: FtrProviderContext) { modelMemory: '5mb', createIndexPattern: true, expected: { + histogramCharts: [ + { chartAvailable: true, id: '1stFlrSF', legend: '334 - 4692' }, + { chartAvailable: true, id: 'BsmtFinSF1', legend: '0 - 5644' }, + { chartAvailable: true, id: 'BsmtQual', legend: '0 - 5' }, + { chartAvailable: true, id: 'CentralAir', legend: '2 categories' }, + { chartAvailable: true, id: 'Condition2', legend: '2 categories' }, + { chartAvailable: true, id: 'Electrical', legend: '2 categories' }, + { chartAvailable: true, id: 'ExterQual', legend: '1 - 4' }, + { chartAvailable: true, id: 'Exterior1st', legend: '2 categories' }, + { chartAvailable: true, id: 'Exterior2nd', legend: '3 categories' }, + { chartAvailable: true, id: 'Fireplaces', legend: '0 - 3' }, + ], row: { type: 'outlier_detection', status: 'stopped', @@ -84,6 +96,16 @@ export default function ({ getService }: FtrProviderContext) { await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewExists(); }); + it('enables the source data preview histogram charts', async () => { + await ml.dataFrameAnalyticsCreation.enableSourceDataPreviewHistogramCharts(); + }); + + it('displays the source data preview histogram charts', async () => { + await ml.dataFrameAnalyticsCreation.assertSourceDataPreviewHistogramCharts( + testData.expected.histogramCharts + ); + }); + it('displays the include fields selection', async () => { await ml.dataFrameAnalyticsCreation.assertIncludeFieldsSelectionExists(); }); diff --git a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts index 1b756bbaca5d8..fc4aaa4fbf5fd 100644 --- a/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts +++ b/x-pack/test/functional/services/ml/data_frame_analytics_creation.ts @@ -128,6 +128,58 @@ export function MachineLearningDataFrameAnalyticsCreationProvider( await testSubjects.existOrFail('mlAnalyticsCreationDataGrid loaded', { timeout: 5000 }); }, + async assertIndexPreviewHistogramChartButtonExists() { + await testSubjects.existOrFail('mlAnalyticsCreationDataGridHistogramButton'); + }, + + async enableSourceDataPreviewHistogramCharts() { + await this.assertSourceDataPreviewHistogramChartButtonCheckState(false); + await testSubjects.click('mlAnalyticsCreationDataGridHistogramButton'); + await this.assertSourceDataPreviewHistogramChartButtonCheckState(true); + }, + + async assertSourceDataPreviewHistogramChartButtonCheckState(expectedCheckState: boolean) { + const actualCheckState = + (await testSubjects.getAttribute( + 'mlAnalyticsCreationDataGridHistogramButton', + 'aria-checked' + )) === 'true'; + expect(actualCheckState).to.eql( + expectedCheckState, + `Chart histogram button check state should be '${expectedCheckState}' (got '${actualCheckState}')` + ); + }, + + async assertSourceDataPreviewHistogramCharts( + expectedHistogramCharts: Array<{ chartAvailable: boolean; id: string; legend: string }> + ) { + // For each chart, get the content of each header cell and assert + // the legend text and column id and if the chart should be present or not. + await retry.tryForTime(5000, async () => { + for (const [index, expected] of expectedHistogramCharts.entries()) { + await testSubjects.existOrFail(`mlDataGridChart-${index}`); + + if (expected.chartAvailable) { + await testSubjects.existOrFail(`mlDataGridChart-${index}-histogram`); + } else { + await testSubjects.missingOrFail(`mlDataGridChart-${index}-histogram`); + } + + const actualLegend = await testSubjects.getVisibleText(`mlDataGridChart-${index}-legend`); + expect(actualLegend).to.eql( + expected.legend, + `Legend text for column '${index}' should be '${expected.legend}' (got '${actualLegend}')` + ); + + const actualId = await testSubjects.getVisibleText(`mlDataGridChart-${index}-id`); + expect(actualId).to.eql( + expected.id, + `Id text for column '${index}' should be '${expected.id}' (got '${actualId}')` + ); + } + }); + }, + async assertIncludeFieldsSelectionExists() { await testSubjects.existOrFail('mlAnalyticsCreateJobWizardIncludesSelect', { timeout: 5000 }); },