From 53e2d5d7254845399a65e769f1961cccb2524d36 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Thu, 15 Apr 2021 12:52:09 -0400 Subject: [PATCH] [ML] Data Frame Analytics: Ensure creation and results views display nested fields correctly (#96905) * create analytics field service * remove unnecessary field filter. update types * create common base class for newJobCapabilites classes for AD and DFA * fix column schema for histogram * update endpoint to be consistent with AD job caps * add unit test for removeNestedFieldChildren helper function * removes obsolete const --- x-pack/plugins/ml/common/types/fields.ts | 4 + .../components/data_grid/common.ts | 6 +- .../data_frame_analytics/common/fields.ts | 58 +- .../common/get_index_fields.ts | 4 +- .../common/use_results_view_config.ts | 4 +- .../configuration_step_form.tsx | 4 +- .../supported_fields_message.tsx | 10 +- .../hooks/use_index_data.ts | 67 +- .../common/job_creator/util/general.ts | 2 +- .../components/time_field/time_field.tsx | 2 +- .../advanced_view/metric_selection.tsx | 2 +- .../categorization_field.tsx | 2 +- .../categorization_per_partition_dropdown.tsx | 2 +- .../components/influencers/influencers.tsx | 2 +- .../multi_metric_view/metric_selection.tsx | 2 +- .../population_view/metric_selection.tsx | 2 +- .../single_metric_view/metric_selection.tsx | 2 +- .../components/split_field/by_field.tsx | 2 +- .../components/split_field/split_field.tsx | 2 +- .../summary_count_field.tsx | 2 +- .../jobs/new_job/pages/new_job/page.tsx | 2 +- .../jobs/new_job/pages/new_job/wizard.tsx | 2 +- .../analytics_job_creation.tsx | 8 +- .../routing/routes/new_job/wizard.tsx | 8 +- .../nested_field_index_response.json | 611 ++++++++++++++++++ .../ml_api_service/data_frame_analytics.ts | 9 + .../services/new_job_capabilities/index.ts | 8 + .../load_new_job_capabilities.ts | 49 ++ .../new_job_capabilities._service.test.ts | 6 +- .../new_job_capabilities.ts | 65 ++ .../new_job_capabilities_service.ts | 95 +-- .../new_job_capabilities_service_analytics.ts | 66 ++ .../remove_nested_field_children.test.ts | 33 + .../job_service/new_job_caps/field_service.ts | 11 +- .../job_service/new_job_caps/new_job_caps.ts | 6 +- x-pack/plugins/ml/server/routes/apidoc.json | 1 + .../ml/server/routes/data_frame_analytics.ts | 68 ++ .../routes/schemas/data_analytics_schema.ts | 6 + 38 files changed, 1044 insertions(+), 191 deletions(-) create mode 100644 x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json create mode 100644 x-pack/plugins/ml/public/application/services/new_job_capabilities/index.ts create mode 100644 x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts rename x-pack/plugins/ml/public/application/services/{ => new_job_capabilities}/new_job_capabilities._service.test.ts (91%) create mode 100644 x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts rename x-pack/plugins/ml/public/application/services/{ => new_job_capabilities}/new_job_capabilities_service.ts (64%) create mode 100644 x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts create mode 100644 x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts index 45fcfac7e930c..cf017192353e9 100644 --- a/x-pack/plugins/ml/common/types/fields.ts +++ b/x-pack/plugins/ml/common/types/fields.ts @@ -50,6 +50,10 @@ export interface NewJobCaps { aggs: Aggregation[]; } +export interface NewJobCapsResponse { + [indexPattern: string]: NewJobCaps; +} + export interface AggFieldPair { agg: Aggregation; field: Field; diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts index f723c1d72b818..b897ca3dccc51 100644 --- a/x-pack/plugins/ml/public/application/components/data_grid/common.ts +++ b/x-pack/plugins/ml/public/application/components/data_grid/common.ts @@ -103,7 +103,7 @@ export function getCombinedRuntimeMappings( ): RuntimeMappings | undefined { let combinedRuntimeMappings = {}; - // And runtime field mappings defined by index pattern + // Add runtime field mappings defined by index pattern if (indexPattern) { const computedFields = indexPattern?.getComputedFields(); if (computedFields?.runtimeFields !== undefined) { @@ -147,6 +147,7 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results case 'date': schema = 'datetime'; break; + case 'nested': case 'geo_point': schema = 'json'; break; @@ -238,6 +239,9 @@ export const getDataGridSchemaFromKibanaFieldType = ( case KBN_FIELD_TYPES.NUMBER: schema = 'numeric'; break; + case KBN_FIELD_TYPES.NESTED: + schema = 'json'; + break; } if (schema === undefined && field?.aggregatable === false) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts index e62f7dfdf3a91..fee610797577f 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts @@ -16,7 +16,7 @@ import { isRegressionAnalysis, } from '../../../../common/util/analytics_utils'; import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; -import { newJobCapsService } from '../../services/new_job_capabilities_service'; +import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants'; import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics'; @@ -54,18 +54,18 @@ export const ML__ID_COPY = 'ml__id_copy'; export const ML__INCREMENTAL_ID = 'ml__incremental_id'; export const isKeywordAndTextType = (fieldName: string): boolean => { - const { fields } = newJobCapsService; + const { fields } = newJobCapsServiceAnalytics; const fieldType = fields.find((field) => field.name === fieldName)?.type; let isBothTypes = false; // If it's a keyword type - check if it has a corresponding text type if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.KEYWORD) { - const field = newJobCapsService.getFieldById(fieldName.replace(/\.keyword$/, '')); + const field = newJobCapsServiceAnalytics.getFieldById(fieldName.replace(/\.keyword$/, '')); isBothTypes = field !== null && field.type === ES_FIELD_TYPES.TEXT; } else if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.TEXT) { // If text, check if has corresponding keyword type - const field = newJobCapsService.getFieldById(`${fieldName}.keyword`); + const field = newJobCapsServiceAnalytics.getFieldById(`${fieldName}.keyword`); isBothTypes = field !== null && field.type === ES_FIELD_TYPES.KEYWORD; } @@ -180,24 +180,22 @@ export const getDefaultFieldsFromJobCaps = ( // default is 'ml' const resultsField = jobConfig.dest.results_field; - const featureImportanceFields = []; - const topClassesFields = []; const allFields: any = []; let type: ES_FIELD_TYPES | undefined; let predictedField: string | undefined; if (isOutlierAnalysis(jobConfig.analysis)) { - if (jobConfig.analysis.outlier_detection.compute_feature_influence) { - featureImportanceFields.push({ - id: `${resultsField}.${FEATURE_INFLUENCE}`, - name: `${resultsField}.${FEATURE_INFLUENCE}`, - type: KBN_FIELD_TYPES.UNKNOWN, - }); + if (!jobConfig.analysis.outlier_detection.compute_feature_influence) { + // remove all feature influence fields + fields = fields.filter( + (field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}`) + ); + } else { + // remove flattened feature influence fields + fields = fields.filter( + (field: any) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`) + ); } - // remove flattened feature influence fields - fields = fields.filter( - (field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`) - ); // Only need to add these fields if we didn't use dest index pattern to get the fields if (needsDestIndexFields === true) { @@ -211,7 +209,7 @@ export const getDefaultFieldsFromJobCaps = ( if (isClassificationAnalysis(jobConfig.analysis) || isRegressionAnalysis(jobConfig.analysis)) { const dependentVariable = getDependentVar(jobConfig.analysis); - type = newJobCapsService.getFieldById(dependentVariable)?.type; + type = newJobCapsServiceAnalytics.getFieldById(dependentVariable)?.type; const predictionFieldName = getPredictionFieldName(jobConfig.analysis); const numTopFeatureImportanceValues = getNumTopFeatureImportanceValues(jobConfig.analysis); const numTopClasses = getNumTopClasses(jobConfig.analysis); @@ -221,24 +219,24 @@ export const getDefaultFieldsFromJobCaps = ( predictionFieldName ? predictionFieldName : defaultPredictionField }`; - if ((numTopFeatureImportanceValues ?? 0) > 0) { - featureImportanceFields.push({ - id: `${resultsField}.${FEATURE_IMPORTANCE}`, - name: `${resultsField}.${FEATURE_IMPORTANCE}`, - type: KBN_FIELD_TYPES.UNKNOWN, - }); + if ((numTopFeatureImportanceValues ?? 0) === 0) { + // remove all feature importance fields + fields = fields.filter( + (field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}`) + ); + } else { // remove flattened feature importance fields fields = fields.filter( (field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}.`) ); } - if ((numTopClasses ?? 0) > 0) { - topClassesFields.push({ - id: `${resultsField}.${TOP_CLASSES}`, - name: `${resultsField}.${TOP_CLASSES}`, - type: KBN_FIELD_TYPES.UNKNOWN, - }); + if ((numTopClasses ?? 0) === 0) { + // remove all top classes fields + fields = fields.filter( + (field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}`) + ); + } else { // remove flattened top classes fields fields = fields.filter( (field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}.`) @@ -258,7 +256,7 @@ export const getDefaultFieldsFromJobCaps = ( } } - allFields.push(...fields, ...featureImportanceFields, ...topClassesFields); + allFields.push(...fields); allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) => sortExplorationResultsFields(a, b, jobConfig) ); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts index b3ad553c9aad2..58146474815c4 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts @@ -7,7 +7,7 @@ import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; -import { newJobCapsService } from '../../services/new_job_capabilities_service'; +import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { getDefaultFieldsFromJobCaps, DataFrameAnalyticsConfig } from '../common'; @@ -19,7 +19,7 @@ export const getIndexFields = ( jobConfig: DataFrameAnalyticsConfig | undefined, needsDestIndexFields: boolean ) => { - const { fields } = newJobCapsService; + const { fields } = newJobCapsServiceAnalytics; if (jobConfig !== undefined) { const { selectedFields: defaultSelected, docFields } = getDefaultFieldsFromJobCaps( fields, diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts index 730b3b632398f..e1f0db4e9291c 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts @@ -15,7 +15,7 @@ import { extractErrorMessage } from '../../../../common/util/errors'; import { getIndexPatternIdFromName } from '../../util/index_utils'; import { ml } from '../../services/ml_api_service'; -import { newJobCapsService } from '../../services/new_job_capabilities_service'; +import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { useMlContext } from '../../contexts/ml'; import { DataFrameAnalyticsConfig } from '../common'; @@ -125,7 +125,7 @@ export const useResultsViewConfig = (jobId: string) => { } if (indexP !== undefined) { - await newJobCapsService.initializeFromIndexPattern(indexP, false, false); + await newJobCapsServiceAnalytics.initializeFromIndexPattern(indexP); setJobConfig(analyticsConfigs.data_frame_analytics[0]); setIndexPattern(indexP); setIsInitialized(true); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx index 1046f1a8c3e92..810f59d904696 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx @@ -19,7 +19,7 @@ import { import { i18n } from '@kbn/i18n'; import { debounce, cloneDeep } from 'lodash'; -import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; +import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { useMlContext } from '../../../../../contexts/ml'; import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common'; @@ -196,7 +196,7 @@ export const ConfigurationStepForm: FC = ({ const depVarOptions = []; let depVarUpdate = formState.dependentVariable; // Get fields and filter for supported types for job type - const { fields } = newJobCapsService; + const { fields } = newJobCapsServiceAnalytics; let resetDependentVariable = true; for (const field of fields) { diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx index 24e1e01b52124..eae1d240be8af 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx @@ -16,7 +16,7 @@ import { OMIT_FIELDS } from '../../../../../../../common/constants/field_types'; import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields'; import { CATEGORICAL_TYPES } from './form_options_validation'; import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public'; -import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; +import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics'; const containsClassificationFieldsCb = ({ name, type }: Field) => @@ -32,7 +32,9 @@ const containsRegressionFieldsCb = ({ name, type }: Field) => (BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type)); const containsOutlierFieldsCb = ({ name, type }: Field) => - !OMIT_FIELDS.includes(name) && name !== EVENT_RATE_FIELD_ID && BASIC_NUMERICAL_TYPES.has(type); + !OMIT_FIELDS.includes(name) && + name !== EVENT_RATE_FIELD_ID && + (BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type)); const callbacks: Record boolean> = { [ANALYSIS_CONFIG_TYPE.CLASSIFICATION]: containsClassificationFieldsCb, @@ -71,7 +73,7 @@ export const SupportedFieldsMessage: FC = ({ jobType }) => { setSourceIndexContainsSupportedFields, ] = useState(true); const [sourceIndexFieldsCheckFailed, setSourceIndexFieldsCheckFailed] = useState(false); - const { fields } = newJobCapsService; + const { fields } = newJobCapsServiceAnalytics; // Find out if index pattern contains supported fields for job type. Provides a hint in the form // that job may not run correctly if no supported fields are found. @@ -90,8 +92,6 @@ export const SupportedFieldsMessage: FC = ({ jobType }) => { useEffect(() => { if (jobType !== undefined) { - setSourceIndexContainsSupportedFields(true); - setSourceIndexFieldsCheckFailed(false); validateFields(); } }, [jobType]); diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts index 2d9ae1cd4689b..2ae75083bff43 100644 --- a/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts +++ b/x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts @@ -15,6 +15,7 @@ import { IndexPattern } from '../../../../../../../../../src/plugins/data/public import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils'; import { RuntimeMappings } from '../../../../../../common/types/fields'; import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms'; +import { newJobCapsServiceAnalytics } from '../../../../services/new_job_capabilities/new_job_capabilities_service_analytics'; import { DataLoader } from '../../../../datavisualizer/index_based/data_loader'; @@ -49,6 +50,41 @@ function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) { }); } +function getInitialColumns(indexPattern: IndexPattern) { + const { fields } = newJobCapsServiceAnalytics; + const columns = fields.map((field: any) => { + const schema = + getDataGridSchemaFromESFieldType(field.type) || getDataGridSchemaFromKibanaFieldType(field); + + return { + id: field.name, + schema, + isExpandable: schema !== 'boolean', + isRuntimeFieldColumn: false, + }; + }); + + // Add runtime fields defined in index pattern to columns + if (indexPattern) { + const computedFields = indexPattern?.getComputedFields(); + + if (isRuntimeMappings(computedFields.runtimeFields)) { + Object.keys(computedFields.runtimeFields).forEach((runtimeField) => { + const schema = getDataGridSchemaFromESFieldType( + computedFields.runtimeFields[runtimeField].type + ); + columns.push({ + id: runtimeField, + schema, + isExpandable: schema !== 'boolean', + isRuntimeFieldColumn: true, + }); + }); + } + } + return columns; +} + export const useIndexData = ( indexPattern: IndexPattern, query: Record | undefined, @@ -58,23 +94,7 @@ export const useIndexData = ( const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [ indexPattern, ]); - - const [columns, setColumns] = useState([ - ...indexPatternFields.map((id) => { - const field = indexPattern.fields.getByName(id); - const isRuntimeFieldColumn = field?.runtimeField !== undefined; - const schema = isRuntimeFieldColumn - ? getDataGridSchemaFromESFieldType(field?.type as estypes.RuntimeField['type']) - : getDataGridSchemaFromKibanaFieldType(field); - return { - id, - schema, - isExpandable: schema !== 'boolean', - isRuntimeFieldColumn, - }; - }), - ]); - + const [columns, setColumns] = useState(getInitialColumns(indexPattern)); const dataGrid = useDataGrid(columns); const { @@ -131,18 +151,7 @@ export const useIndexData = ( ...(combinedRuntimeMappings ? getRuntimeFieldColumns(combinedRuntimeMappings) : []), ]); } else { - setColumns([ - ...indexPatternFields.map((id) => { - const field = indexPattern.fields.getByName(id); - const schema = getDataGridSchemaFromKibanaFieldType(field); - return { - id, - schema, - isExpandable: schema !== 'boolean', - isRuntimeFieldColumn: field?.runtimeField !== undefined, - }; - }), - ]); + setColumns(getInitialColumns(indexPattern)); } setRowCount(typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total.value); setRowCountRelation( diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index 9ae8585933ca8..3f306f9bcc996 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs'; -import { newJobCapsService } from '../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { NavigateToPath } from '../../../../../contexts/kibana'; import { ML_JOB_AGGREGATION, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx index ad0f7ee072f09..a25e3b8097bd9 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx @@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { TimeFieldSelect } from './time_field_select'; import { JobCreatorContext } from '../../../job_creator_context'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { AdvancedJobCreator } from '../../../../../common/job_creator'; import { Description } from './description'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx index 8f477043b3908..b4508af7803dd 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx @@ -9,7 +9,7 @@ import React, { Fragment, FC, useContext, useState } from 'react'; import { JobCreatorContext } from '../../../job_creator_context'; import { AdvancedJobCreator } from '../../../../../common/job_creator'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { Aggregation, Field } from '../../../../../../../../../common/types/fields'; import { MetricSelector } from './metric_selector'; import { RichDetector } from '../../../../../common/job_creator/advanced_job_creator'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx index 18b10320b7e2f..6a441a909692b 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_field/categorization_field.tsx @@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { CategorizationFieldSelect } from './categorization_field_select'; import { JobCreatorContext } from '../../../job_creator_context'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { AdvancedJobCreator, CategorizationJobCreator, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx index 7ea7d09d06b34..b404d8238fd5f 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx @@ -10,7 +10,7 @@ import { EuiFormRow } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { CategorizationPerPartitionFieldSelect } from './categorization_per_partition_input'; export const CategorizationPerPartitionFieldDropdown = ({ diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx index 091864cf330ef..5050a4f3be8ea 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx @@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { InfluencersSelect } from './influencers_select'; import { JobCreatorContext } from '../../../job_creator_context'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { MultiMetricJobCreator, PopulationJobCreator, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx index 0515496469030..6cbf96fcb04e8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/multi_metric_view/metric_selection.tsx @@ -11,7 +11,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { MultiMetricJobCreator } from '../../../../../common/job_creator'; import { LineChartData } from '../../../../../common/chart_loader'; import { DropDownLabel, DropDownProps } from '../agg_select'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { AggFieldPair } from '../../../../../../../../../common/types/fields'; import { sortFields } from '../../../../../../../../../common/util/fields_utils'; import { getChartSettings, defaultChartSettings } from '../../../charts/common/settings'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx index 7f5a06925c7e8..c5efd4b226d5c 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/population_view/metric_selection.tsx @@ -12,7 +12,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { PopulationJobCreator } from '../../../../../common/job_creator'; import { LineChartData } from '../../../../../common/chart_loader'; import { DropDownLabel, DropDownProps } from '../agg_select'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { Field, AggFieldPair } from '../../../../../../../../../common/types/fields'; import { sortFields } from '../../../../../../../../../common/util/fields_utils'; import { getChartSettings, defaultChartSettings } from '../../../charts/common/settings'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx index c5c5cd4d8b744..b2a97d8e0589a 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/single_metric_view/metric_selection.tsx @@ -10,7 +10,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { SingleMetricJobCreator } from '../../../../../common/job_creator'; import { LineChartData } from '../../../../../common/chart_loader'; import { AggSelect, DropDownLabel, DropDownProps, createLabel } from '../agg_select'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { AggFieldPair } from '../../../../../../../../../common/types/fields'; import { sortFields } from '../../../../../../../../../common/util/fields_utils'; import { AnomalyChart, CHART_TYPE } from '../../../charts/anomaly_chart'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/by_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/by_field.tsx index 01c538f7ceb01..b197b950bbe28 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/by_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/by_field.tsx @@ -14,7 +14,7 @@ import { Field } from '../../../../../../../../../common/types/fields'; import { newJobCapsService, filterCategoryFields, -} from '../../../../../../../services/new_job_capabilities_service'; +} from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { MultiMetricJobCreator, PopulationJobCreator } from '../../../../../common/job_creator'; interface Props { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/split_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/split_field.tsx index 7a99d4da13185..9837fe924fb01 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/split_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/split_field/split_field.tsx @@ -12,7 +12,7 @@ import { JobCreatorContext } from '../../../job_creator_context'; import { newJobCapsService, filterCategoryFields, -} from '../../../../../../../services/new_job_capabilities_service'; +} from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { Description } from './description'; import { MultiMetricJobCreator, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx index 39ec89f8424fc..3345b60ddd4d9 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/summary_count_field/summary_count_field.tsx @@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { SummaryCountFieldSelect } from './summary_count_field_select'; import { JobCreatorContext } from '../../../job_creator_context'; -import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service'; import { MultiMetricJobCreator, PopulationJobCreator, diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx index 3a01ce8c70fc8..442bdba717f28 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/page.tsx @@ -37,7 +37,7 @@ import { useMlContext } from '../../../../contexts/ml'; import { getTimeFilterRange } from '../../../../components/full_time_range_selector'; import { getTimeBucketsFromCache } from '../../../../util/time_buckets'; import { ExistingJobsAndGroups, mlJobService } from '../../../../services/job_service'; -import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../services/new_job_capabilities/new_job_capabilities_service'; import { EVENT_RATE_FIELD_ID } from '../../../../../../common/types/fields'; import { getNewJobDefaults } from '../../../../services/ml_server_info'; import { useToastNotificationService } from '../../../../services/toast_notification_service'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard.tsx index 5764e0fee511b..06d532e4bd793 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/new_job/wizard.tsx @@ -19,7 +19,7 @@ import { JobCreatorType } from '../../common/job_creator'; import { ChartLoader } from '../../common/chart_loader'; import { ResultsLoader } from '../../common/results_loader'; import { JobValidator } from '../../common/job_validator'; -import { newJobCapsService } from '../../../../services/new_job_capabilities_service'; +import { newJobCapsService } from '../../../../services/new_job_capabilities/new_job_capabilities_service'; import { WizardSteps } from './wizard_steps'; import { WizardHorizontalSteps } from './wizard_horizontal_steps'; import { JOB_TYPE } from '../../../../../../common/constants/new_job'; diff --git a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx index 7c011952edd68..ab57c264683ca 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/data_frame_analytics/analytics_job_creation.tsx @@ -17,7 +17,10 @@ import { useResolver } from '../../use_resolver'; import { basicResolvers } from '../../resolvers'; import { Page } from '../../../data_frame_analytics/pages/analytics_creation'; import { breadcrumbOnClickFactory, getBreadcrumbWithUrlForApp } from '../../breadcrumbs'; -import { loadNewJobCapabilities } from '../../../services/new_job_capabilities_service'; +import { + loadNewJobCapabilities, + DATA_FRAME_ANALYTICS, +} from '../../../services/new_job_capabilities/load_new_job_capabilities'; export const analyticsJobsCreationRouteFactory = ( navigateToPath: NavigateToPath, @@ -43,7 +46,8 @@ const PageWrapper: FC = ({ location, deps }) => { const { context } = useResolver(index, savedSearchId, deps.config, { ...basicResolvers(deps), - jobCaps: () => loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns), + analyticsFields: () => + loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns, DATA_FRAME_ANALYTICS), }); return ( diff --git a/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx b/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx index 00f3e85343553..726ec328d1cb2 100644 --- a/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx +++ b/x-pack/plugins/ml/public/application/routing/routes/new_job/wizard.tsx @@ -17,7 +17,10 @@ import { useResolver } from '../../use_resolver'; import { Page } from '../../../jobs/new_job/pages/new_job'; import { JOB_TYPE } from '../../../../../common/constants/new_job'; import { mlJobService } from '../../../services/job_service'; -import { loadNewJobCapabilities } from '../../../services/new_job_capabilities_service'; +import { + loadNewJobCapabilities, + ANOMALY_DETECTOR, +} from '../../../services/new_job_capabilities/load_new_job_capabilities'; import { checkCreateJobsCapabilitiesResolver } from '../../../capabilities/check_capabilities'; import { getBreadcrumbWithUrlForApp } from '../../breadcrumbs'; import { useCreateAndNavigateToMlLink } from '../../../contexts/kibana/use_create_url'; @@ -137,7 +140,8 @@ const PageWrapper: FC = ({ location, jobType, deps }) => { const { context, results } = useResolver(index, savedSearchId, deps.config, { ...basicResolvers(deps), privileges: () => checkCreateJobsCapabilitiesResolver(redirectToJobsManagementPage), - jobCaps: () => loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns), + jobCaps: () => + loadNewJobCapabilities(index, savedSearchId, deps.indexPatterns, ANOMALY_DETECTOR), existingJobsAndGroups: mlJobService.getJobAndGroupIds, }); diff --git a/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json b/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json new file mode 100644 index 0000000000000..49baa9fb7e427 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/__mocks__/nested_field_index_response.json @@ -0,0 +1,611 @@ +{ + "nested-field-index":{ + "aggs":[ + { + "id":"count", + "title":"Count", + "kibanaName":"count", + "dslName":"count", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"high_count", + "title":"High count", + "kibanaName":"count", + "dslName":"count", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"low_count", + "title":"Low count", + "kibanaName":"count", + "dslName":"count", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"mean", + "title":"Mean", + "kibanaName":"avg", + "dslName":"avg", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"avg", + "min":"avg" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"high_mean", + "title":"High mean", + "kibanaName":"avg", + "dslName":"avg", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"avg", + "min":"avg" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"low_mean", + "title":"Low mean", + "kibanaName":"avg", + "dslName":"avg", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"avg", + "min":"avg" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"sum", + "title":"Sum", + "kibanaName":"sum", + "dslName":"sum", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"sum", + "min":"sum" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"high_sum", + "title":"High sum", + "kibanaName":"sum", + "dslName":"sum", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"sum", + "min":"sum" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"low_sum", + "title":"Low sum", + "kibanaName":"sum", + "dslName":"sum", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"sum", + "min":"sum" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"median", + "title":"Median", + "kibanaName":"median", + "dslName":"percentiles", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds": [ + "time_spent" + ] + }, + { + "id":"high_median", + "title":"High median", + "kibanaName":"median", + "dslName":"percentiles", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"low_median", + "title":"Low median", + "kibanaName":"median", + "dslName":"percentiles", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"min", + "title":"Min", + "kibanaName":"min", + "dslName":"min", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"min", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"max", + "title":"Max", + "kibanaName":"max", + "dslName":"max", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"max" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"distinct_count", + "title":"Distinct count", + "kibanaName":"cardinality", + "dslName":"cardinality", + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"non_zero_count", + "title":"Non zero count", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"high_non_zero_count", + "title":"High non zero count", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"low_non_zero_count", + "title":"Low non zero count", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"high_distinct_count", + "title":"High distinct count", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"low_distinct_count", + "title":"Low distinct count", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"metric", + "title":"Metric", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"varp", + "title":"varp", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"high_varp", + "title":"High varp", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"low_varp", + "title":"Low varp", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"non_null_sum", + "title":"Non null sum", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"high_non_null_sum", + "title":"High non null sum", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"low_non_null_sum", + "title":"Low non null sum", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "time_spent" + ] + }, + { + "id":"rare", + "title":"Rare", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"freq_rare", + "title":"Freq rare", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"info_content", + "title":"Info content", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group", + "user.first", + "user.last", + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"high_info_content", + "title":"High info content", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group", + "user.first", + "user.last", + "group.keyword", + "usr.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"low_info_content", + "title":"Low info content", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + "group", + "user.first", + "user.last", + "group.keyword", + "user.first.keyword", + "user.last.keyword", + "time_spent" + ] + }, + { + "id":"time_of_day", + "title":"Time of day", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"time_of_week", + "title":"Time of week", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + } + }, + { + "id":"lat_long", + "title":"Lat long", + "kibanaName":null, + "dslName":null, + "type":"metrics", + "mlModelPlotAgg":{ + "max":"max", + "min":"min" + }, + "fieldIds":[ + + ] + } + ], + "fields":[ + { + "id":"group", + "name":"group", + "type":"text", + "aggregatable":false, + "aggIds":[ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"group.keyword", + "name":"group.keyword", + "type":"keyword", + "aggregatable":true, + "aggIds":[ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"time_spent", + "name":"time_spent", + "type":"long", + "aggregatable":true, + "aggIds":[ + "mean", + "high_mean", + "low_mean", + "sum", + "high_sum", + "low_sum", + "median", + "high_median", + "low_median", + "min", + "max", + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "metric", + "varp", + "high_varp", + "low_varp", + "non_null_sum", + "high_non_null_sum", + "low_non_null_sum", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"user", + "name":"user", + "type":"nested", + "aggregatable":false, + "aggIds":[ + + ] + }, + { + "id":"user.first", + "name":"user.first", + "type":"text", + "aggregatable":false, + "aggIds":[ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"user.first.keyword", + "name":"user.first.keyword", + "type":"keyword", + "aggregatable":true, + "aggIds":[ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"user.last", + "name":"user.last", + "type":"text", + "aggregatable":false, + "aggIds":[ + "info_content", + "high_info_content", + "low_info_content" + ] + }, + { + "id":"user.last.keyword", + "name":"user.last.keyword", + "type":"keyword", + "aggregatable":true, + "aggIds":[ + "distinct_count", + "high_distinct_count", + "low_distinct_count", + "info_content", + "high_info_content", + "low_info_content" + ] + } + ] + } +} diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts index 8fe34932ffa88..39662cfedd901 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/data_frame_analytics.ts @@ -15,6 +15,7 @@ import { UpdateDataFrameAnalyticsConfig, } from '../../data_frame_analytics/common'; import { DeepPartial } from '../../../../common/types/common'; +import { NewJobCapsResponse } from '../../../../common/types/fields'; import { DeleteDataFrameAnalyticsWithIndexStatus, AnalyticsMapReturnType, @@ -175,4 +176,12 @@ export const dataFrameAnalytics = { body, }); }, + newJobCapsAnalytics(indexPatternTitle: string, isRollup: boolean = false) { + const query = isRollup === true ? { rollup: true } : {}; + return http({ + path: `${basePath()}/data_frame/analytics/new_job_caps/${indexPatternTitle}`, + method: 'GET', + query, + }); + }, }; diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/index.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/index.ts new file mode 100644 index 0000000000000..d4d01143b8a69 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { NewJobCapabilitiesServiceBase, processTextAndKeywordFields } from './new_job_capabilities'; diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts new file mode 100644 index 0000000000000..a998343535249 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/load_new_job_capabilities.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IIndexPattern, IndexPatternsContract } from '../../../../../../../src/plugins/data/public'; +import { getIndexPatternAndSavedSearch } from '../../util/index_utils'; +import { JobType } from '../../../../common/types/saved_objects'; +import { newJobCapsServiceAnalytics } from '../new_job_capabilities/new_job_capabilities_service_analytics'; +import { newJobCapsService } from '../new_job_capabilities/new_job_capabilities_service'; + +export const ANOMALY_DETECTOR = 'anomaly-detector'; +export const DATA_FRAME_ANALYTICS = 'data-frame-analytics'; + +// called in the routing resolve block to initialize the NewJobCapabilites +// service for the corresponding job type with the currently selected index pattern +export function loadNewJobCapabilities( + indexPatternId: string, + savedSearchId: string, + indexPatterns: IndexPatternsContract, + jobType: JobType +) { + return new Promise(async (resolve, reject) => { + const serviceToUse = + jobType === ANOMALY_DETECTOR ? newJobCapsService : newJobCapsServiceAnalytics; + if (indexPatternId !== undefined) { + // index pattern is being used + const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId); + await serviceToUse.initializeFromIndexPattern(indexPattern); + resolve(serviceToUse.newJobCaps); + } else if (savedSearchId !== undefined) { + // saved search is being used + // load the index pattern from the saved search + const { indexPattern } = await getIndexPatternAndSavedSearch(savedSearchId); + if (indexPattern === null) { + // eslint-disable-next-line no-console + console.error('Cannot retrieve index pattern from saved search'); + reject(); + return; + } + await serviceToUse.initializeFromIndexPattern(indexPattern); + resolve(serviceToUse.newJobCaps); + } else { + reject(); + } + }); +} diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities._service.test.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts similarity index 91% rename from x-pack/plugins/ml/public/application/services/new_job_capabilities._service.test.ts rename to x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts index 217344366b67a..4c0ee7b67a994 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities._service.test.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities._service.test.ts @@ -6,13 +6,13 @@ */ import { newJobCapsService } from './new_job_capabilities_service'; -import { IndexPattern } from '../../../../../../src/plugins/data/public'; +import { IndexPattern } from '../../../../../../../src/plugins/data/public'; // there is magic happening here. starting the include name with `mock..` // ensures it can be lazily loaded by the jest.mock function below. -import mockCloudwatchResponse from './__mocks__/cloudwatch_job_caps_response.json'; +import mockCloudwatchResponse from '../__mocks__/cloudwatch_job_caps_response.json'; -jest.mock('./ml_api_service', () => ({ +jest.mock('../ml_api_service', () => ({ ml: { jobs: { newJobCaps: jest.fn(() => Promise.resolve(mockCloudwatchResponse)), diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts new file mode 100644 index 0000000000000..bbb9b141c7349 --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Aggregation, Field, NewJobCaps } from '../../../../common/types/fields'; +import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public'; + +// create two lists, one removing text fields if there are keyword equivalents and vice versa +export function processTextAndKeywordFields(fields: Field[]) { + const keywordIds = fields.filter((f) => f.type === ES_FIELD_TYPES.KEYWORD).map((f) => f.id); + const textIds = fields.filter((f) => f.type === ES_FIELD_TYPES.TEXT).map((f) => f.id); + + const fieldsPreferringKeyword = fields.filter( + (f) => + f.type !== ES_FIELD_TYPES.TEXT || + (f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false) + ); + + const fieldsPreferringText = fields.filter( + (f) => + f.type !== ES_FIELD_TYPES.KEYWORD || + (f.type === ES_FIELD_TYPES.KEYWORD && + textIds.includes(f.id.replace(/\.keyword$/, '')) === false) + ); + + return { fieldsPreferringKeyword, fieldsPreferringText }; +} + +export class NewJobCapabilitiesServiceBase { + protected _fields: Field[]; + protected _aggs: Aggregation[]; + + constructor() { + this._fields = []; + this._aggs = []; + } + + public get fields(): Field[] { + return this._fields; + } + + public get aggs(): Aggregation[] { + return this._aggs; + } + + public get newJobCaps(): NewJobCaps { + return { + fields: this._fields, + aggs: this._aggs, + }; + } + + public getFieldById(id: string): Field | null { + const field = this._fields.find((f) => f.id === id); + return field === undefined ? null : field; + } + + public getAggById(id: string): Aggregation | null { + const agg = this._aggs.find((f) => f.id === id); + return agg === undefined ? null : agg; + } +} diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities_service.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts similarity index 64% rename from x-pack/plugins/ml/public/application/services/new_job_capabilities_service.ts rename to x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts index b9520df4e710f..9324b3c0f0824 100644 --- a/x-pack/plugins/ml/public/application/services/new_job_capabilities_service.ts +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service.ts @@ -4,68 +4,25 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { Field, Aggregation, AggId, FieldId, - NewJobCaps, EVENT_RATE_FIELD_ID, -} from '../../../common/types/fields'; -import { - ES_FIELD_TYPES, - IIndexPattern, - IndexPatternsContract, -} from '../../../../../../src/plugins/data/public'; -import { ml } from './ml_api_service'; -import { getIndexPatternAndSavedSearch } from '../util/index_utils'; - -// called in the routing resolve block to initialize the -// newJobCapsService with the currently selected index pattern -export function loadNewJobCapabilities( - indexPatternId: string, - savedSearchId: string, - indexPatterns: IndexPatternsContract -) { - return new Promise(async (resolve, reject) => { - if (indexPatternId !== undefined) { - // index pattern is being used - const indexPattern: IIndexPattern = await indexPatterns.get(indexPatternId); - await newJobCapsService.initializeFromIndexPattern(indexPattern); - resolve(newJobCapsService.newJobCaps); - } else if (savedSearchId !== undefined) { - // saved search is being used - // load the index pattern from the saved search - const { indexPattern } = await getIndexPatternAndSavedSearch(savedSearchId); - if (indexPattern === null) { - // eslint-disable-next-line no-console - console.error('Cannot retrieve index pattern from saved search'); - reject(); - return; - } - await newJobCapsService.initializeFromIndexPattern(indexPattern); - resolve(newJobCapsService.newJobCaps); - } else { - reject(); - } - }); -} +} from '../../../../common/types/fields'; +import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public'; +import { ml } from '../ml_api_service'; +import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities'; const categoryFieldTypes = [ES_FIELD_TYPES.TEXT, ES_FIELD_TYPES.KEYWORD, ES_FIELD_TYPES.IP]; -class NewJobCapsService { - private _fields: Field[] = []; +class NewJobCapsService extends NewJobCapabilitiesServiceBase { private _catFields: Field[] = []; private _dateFields: Field[] = []; - private _aggs: Aggregation[] = []; private _includeEventRateField: boolean = true; private _removeTextFields: boolean = true; - public get fields(): Field[] { - return this._fields; - } - public get catFields(): Field[] { return this._catFields; } @@ -74,17 +31,6 @@ class NewJobCapsService { return this._dateFields; } - public get aggs(): Aggregation[] { - return this._aggs; - } - - public get newJobCaps(): NewJobCaps { - return { - fields: this._fields, - aggs: this._aggs, - }; - } - public get categoryFields(): Field[] { return filterCategoryFields(this._fields); } @@ -126,16 +72,6 @@ class NewJobCapsService { console.error('Unable to load new job capabilities', error); // eslint-disable-line no-console } } - - public getFieldById(id: string): Field | null { - const field = this._fields.find((f) => f.id === id); - return field === undefined ? null : field; - } - - public getAggById(id: string): Aggregation | null { - const agg = this._aggs.find((f) => f.id === id); - return agg === undefined ? null : agg; - } } // using the response from the endpoint, create the field and aggs objects @@ -231,27 +167,6 @@ function addEventRateField(aggs: Aggregation[], fields: Field[]) { fields.splice(0, 0, eventRateField); } -// create two lists, one removing text fields if there are keyword equivalents and vice versa -function processTextAndKeywordFields(fields: Field[]) { - const keywordIds = fields.filter((f) => f.type === ES_FIELD_TYPES.KEYWORD).map((f) => f.id); - const textIds = fields.filter((f) => f.type === ES_FIELD_TYPES.TEXT).map((f) => f.id); - - const fieldsPreferringKeyword = fields.filter( - (f) => - f.type !== ES_FIELD_TYPES.TEXT || - (f.type === ES_FIELD_TYPES.TEXT && keywordIds.includes(`${f.id}.keyword`) === false) - ); - - const fieldsPreferringText = fields.filter( - (f) => - f.type !== ES_FIELD_TYPES.KEYWORD || - (f.type === ES_FIELD_TYPES.KEYWORD && - textIds.includes(f.id.replace(/\.keyword$/, '')) === false) - ); - - return { fieldsPreferringKeyword, fieldsPreferringText }; -} - export function filterCategoryFields(fields: Field[]) { return fields.filter((f) => categoryFieldTypes.includes(f.type)); } diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts new file mode 100644 index 0000000000000..3a362a88e40bb --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/new_job_capabilities_service_analytics.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Field, NewJobCapsResponse } from '../../../../common/types/fields'; +import { ES_FIELD_TYPES, IIndexPattern } from '../../../../../../../src/plugins/data/public'; +import { processTextAndKeywordFields, NewJobCapabilitiesServiceBase } from './new_job_capabilities'; +import { ml } from '../ml_api_service'; + +// Keep top nested field and remove all .* fields +export function removeNestedFieldChildren(resp: NewJobCapsResponse, indexPatternTitle: string) { + const results = resp[indexPatternTitle]; + const fields: Field[] = []; + const nestedFields: Record = {}; + + if (results !== undefined) { + results.fields.forEach((field: Field) => { + if (field.type === ES_FIELD_TYPES.NESTED && nestedFields[field.name] === undefined) { + nestedFields[field.name] = true; + fields.push(field); + } + }); + + if (Object.keys(nestedFields).length > 0) { + results.fields.forEach((field: Field) => { + if (field.type !== ES_FIELD_TYPES.NESTED) { + const fieldNameParts = field.name.split('.'); + const rootOfField = fieldNameParts.shift(); + if (rootOfField && nestedFields[rootOfField] === undefined) { + fields.push(field); + } + } + }); + } else { + fields.push(...results.fields); + } + } + return fields; +} + +class NewJobCapsServiceAnalytics extends NewJobCapabilitiesServiceBase { + public async initializeFromIndexPattern(indexPattern: IIndexPattern) { + try { + const resp: NewJobCapsResponse = await ml.dataFrameAnalytics.newJobCapsAnalytics( + indexPattern.title, + indexPattern.type === 'rollup' + ); + + const allFields = removeNestedFieldChildren(resp, indexPattern.title); + + const { fieldsPreferringKeyword } = processTextAndKeywordFields(allFields); + + // set the main fields list to contain fields which have been filtered to prefer + // keyword fields over text fields. + // e.g. if foo.keyword and foo exist, don't add foo to the list. + this._fields = fieldsPreferringKeyword; + } catch (error) { + console.error('Unable to load analytics index fields', error); // eslint-disable-line no-console + } + } +} + +export const newJobCapsServiceAnalytics = new NewJobCapsServiceAnalytics(); diff --git a/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts b/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts new file mode 100644 index 0000000000000..e960095b5b2db --- /dev/null +++ b/x-pack/plugins/ml/public/application/services/new_job_capabilities/remove_nested_field_children.test.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { removeNestedFieldChildren } from './new_job_capabilities_service_analytics'; +import { IndexPattern } from '../../../../../../../src/plugins/data/public'; + +// there is magic happening here. starting the include name with `mock..` +// ensures it can be lazily loaded by the jest.mock function below. +import nestedFieldIndexResponse from '../__mocks__/nested_field_index_response.json'; + +const indexPattern = ({ + id: 'nested-field-index', + title: 'nested-field-index', +} as unknown) as IndexPattern; + +describe('removeNestedFieldChildren', () => { + describe('cloudwatch newJobCapsAnalytics()', () => { + it('can get job caps fields from endpoint json', async () => { + // @ts-ignore + const fields = removeNestedFieldChildren(nestedFieldIndexResponse, indexPattern.title); + const nestedField = fields.find(({ type }) => type === 'nested'); + const nestedFieldRoot = nestedField?.name; + const regex = new RegExp(`^${nestedFieldRoot}\\.`, 'i'); + + expect(fields).toHaveLength(4); + expect(fields.some((field) => field.name.match(regex))).toBe(false); + }); + }); +}); diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts index c6cf608fe1e0b..49c09742985d4 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/field_service.ts @@ -70,7 +70,7 @@ class FieldsService { } // create field object from the results from _field_caps - private async createFields(): Promise { + private async createFields(includeNested: boolean = false): Promise { const fieldCaps = await this.loadFieldCaps(); const fields: Field[] = []; if (fieldCaps && fieldCaps.fields) { @@ -80,7 +80,10 @@ class FieldsService { if (firstKey !== undefined) { const field = fc[firstKey]; // add to the list of fields if the field type can be used by ML - if (supportedTypes.includes(field.type) === true && field.metadata_field !== true) { + if ( + (supportedTypes.includes(field.type) === true && field.metadata_field !== true) || + (includeNested && field.type === ES_FIELD_TYPES.NESTED) + ) { fields.push({ id: k, name: k, @@ -101,7 +104,7 @@ class FieldsService { // based on what is available in the rollup job // the _indexPattern will be replaced with a comma separated list // of index patterns from all of the rollup jobs - public async getData(): Promise { + public async getData(includeNested: boolean = false): Promise { let rollupFields: RollupFields = {}; if (this._isRollup) { @@ -128,7 +131,7 @@ class FieldsService { } const aggs = cloneDeep([...aggregations, ...mlOnlyAggregations]); - const fields: Field[] = await this.createFields(); + const fields: Field[] = await this.createFields(includeNested); return combineFieldsAndAggs(fields, aggs, rollupFields); } diff --git a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts index c85c393e6dba8..6444f9ae3f61a 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job_caps/new_job_caps.ts @@ -7,13 +7,9 @@ import { IScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; import { _DOC_COUNT } from '../../../../common/constants/field_types'; -import { Aggregation, Field, NewJobCaps } from '../../../../common/types/fields'; +import { Aggregation, Field, NewJobCapsResponse } from '../../../../common/types/fields'; import { fieldServiceProvider } from './field_service'; -export interface NewJobCapsResponse { - [indexPattern: string]: NewJobCaps; -} - export function newJobCapsProvider(client: IScopedClusterClient) { async function newJobCaps( indexPattern: string, diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index ba61a987d69ef..55f66b354df27 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -18,6 +18,7 @@ "DeleteDataFrameAnalytics", "JobsExist", "GetDataFrameAnalyticsIdMap", + "AnalyticsNewJobCaps", "ValidateDataFrameAnalytics", "DataVisualizer", diff --git a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts index 520f8ce6fb0a9..d8b88deb50a43 100644 --- a/x-pack/plugins/ml/server/routes/data_frame_analytics.ts +++ b/x-pack/plugins/ml/server/routes/data_frame_analytics.ts @@ -10,6 +10,7 @@ import { wrapError } from '../client/error_wrapper'; import { analyticsAuditMessagesProvider } from '../models/data_frame_analytics/analytics_audit_messages'; import { RouteInitialization } from '../types'; import { JOB_MAP_NODE_TYPES } from '../../common/constants/data_frame_analytics'; +import { Field, Aggregation } from '../../common/types/fields'; import { dataAnalyticsJobConfigSchema, dataAnalyticsJobUpdateSchema, @@ -21,11 +22,14 @@ import { deleteDataFrameAnalyticsJobSchema, jobsExistSchema, analyticsQuerySchema, + analyticsNewJobCapsParamsSchema, + analyticsNewJobCapsQuerySchema, } from './schemas/data_analytics_schema'; import { GetAnalyticsMapArgs, ExtendAnalyticsMapArgs } from '../models/data_frame_analytics/types'; import { IndexPatternHandler } from '../models/data_frame_analytics/index_patterns'; import { AnalyticsManager } from '../models/data_frame_analytics/analytics_manager'; import { validateAnalyticsJob } from '../models/data_frame_analytics/validation'; +import { fieldServiceProvider } from '../models/job_service/new_job_caps/field_service'; import { DeleteDataFrameAnalyticsWithIndexStatus } from '../../common/types/data_frame_analytics'; import { getAuthorizationHeader } from '../lib/request_authorization'; import type { MlClient } from '../lib/ml_client'; @@ -58,6 +62,24 @@ function getExtendedMap( return analytics.extendAnalyticsMapForAnalyticsJob(idOptions); } +// replace the recursive field and agg references with a +// map of ids to allow it to be stringified for transportation +// over the network. +function convertForStringify(aggs: Aggregation[], fields: Field[]): void { + fields.forEach((f) => { + f.aggIds = f.aggs ? f.aggs.map((a) => a.id) : []; + delete f.aggs; + }); + aggs.forEach((a) => { + if (a.fields !== undefined) { + // if the aggregation supports fields, i.e. it's fields list isn't undefined, + // create a list of field ids + a.fieldIds = a.fields.map((f) => f.id); + } + delete a.fields; + }); +} + /** * Routes for the data frame analytics */ @@ -671,6 +693,52 @@ export function dataFrameAnalyticsRoutes({ router, mlLicense, routeGuard }: Rout }) ); + /** + * @apiGroup DataFrameAnalytics + * + * @api {get} api/data_frame/analytics/fields/:indexPattern Get index pattern fields for analytics + * @apiName AnalyticsNewJobCaps + * @apiDescription Retrieve the index fields for analytics + */ + router.get( + { + path: '/api/ml/data_frame/analytics/new_job_caps/{indexPattern}', + validate: { + params: analyticsNewJobCapsParamsSchema, + query: analyticsNewJobCapsQuerySchema, + }, + options: { + tags: ['access:ml:canGetJobs'], + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ client, request, response, context }) => { + try { + const { indexPattern } = request.params; + const isRollup = request.query.rollup === 'true'; + const savedObjectsClient = context.core.savedObjects.client; + const fieldService = fieldServiceProvider( + indexPattern, + isRollup, + client, + savedObjectsClient + ); + const { fields, aggs } = await fieldService.getData(true); + convertForStringify(aggs, fields); + + return response.ok({ + body: { + [indexPattern]: { + aggs, + fields, + }, + }, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); + /** * @apiGroup DataFrameAnalytics * diff --git a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts index 1f5bcbc23423a..31ecd1bdd9ef0 100644 --- a/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/data_analytics_schema.ts @@ -102,3 +102,9 @@ export const jobsExistSchema = schema.object({ export const analyticsMapQuerySchema = schema.maybe( schema.object({ treatAsRoot: schema.maybe(schema.any()), type: schema.maybe(schema.string()) }) ); + +export const analyticsNewJobCapsParamsSchema = schema.object({ indexPattern: schema.string() }); + +export const analyticsNewJobCapsQuerySchema = schema.maybe( + schema.object({ rollup: schema.maybe(schema.string()) }) +);