[ML] Data Frame Analytics: Ensure creation and results views display …

…nested fields correctly (#96905) * create analytics field service * remove unnecessary field filter. update types * create common base class for newJobCapabilites classes for AD and DFA * fix column schema for histogram * update endpoint to be consistent with AD job caps * add unit test for removeNestedFieldChildren helper function * removes obsolete const
elastic · Apr 15, 2021 · 53e2d5d · 53e2d5d
1 parent e11ac98
commit 53e2d5d
Show file tree

Hide file tree

Showing 38 changed files with 1,044 additions and 191 deletions.
diff --git a/x-pack/plugins/ml/common/types/fields.ts b/x-pack/plugins/ml/common/types/fields.ts
@@ -50,6 +50,10 @@ export interface NewJobCaps {
   aggs: Aggregation[];
 }
 
+export interface NewJobCapsResponse {
+  [indexPattern: string]: NewJobCaps;
+}
+
 export interface AggFieldPair {
   agg: Aggregation;
   field: Field;

diff --git a/x-pack/plugins/ml/public/application/components/data_grid/common.ts b/x-pack/plugins/ml/public/application/components/data_grid/common.ts
@@ -103,7 +103,7 @@ export function getCombinedRuntimeMappings(
 ): RuntimeMappings | undefined {
   let combinedRuntimeMappings = {};
 
-  // And runtime field mappings defined by index pattern
+  // Add runtime field mappings defined by index pattern
   if (indexPattern) {
     const computedFields = indexPattern?.getComputedFields();
     if (computedFields?.runtimeFields !== undefined) {
@@ -147,6 +147,7 @@ export const getDataGridSchemasFromFieldTypes = (fieldTypes: FieldTypes, results
       case 'date':
         schema = 'datetime';
         break;
+      case 'nested':
       case 'geo_point':
         schema = 'json';
         break;
@@ -238,6 +239,9 @@ export const getDataGridSchemaFromKibanaFieldType = (
     case KBN_FIELD_TYPES.NUMBER:
       schema = 'numeric';
       break;
+    case KBN_FIELD_TYPES.NESTED:
+      schema = 'json';
+      break;
   }
 
   if (schema === undefined && field?.aggregatable === false) {

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/fields.ts
@@ -16,7 +16,7 @@ import {
   isRegressionAnalysis,
 } from '../../../../common/util/analytics_utils';
 import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
-import { newJobCapsService } from '../../services/new_job_capabilities_service';
+import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 
 import { FEATURE_IMPORTANCE, FEATURE_INFLUENCE, OUTLIER_SCORE, TOP_CLASSES } from './constants';
 import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics';
@@ -54,18 +54,18 @@ export const ML__ID_COPY = 'ml__id_copy';
 export const ML__INCREMENTAL_ID = 'ml__incremental_id';
 
 export const isKeywordAndTextType = (fieldName: string): boolean => {
-  const { fields } = newJobCapsService;
+  const { fields } = newJobCapsServiceAnalytics;
 
   const fieldType = fields.find((field) => field.name === fieldName)?.type;
   let isBothTypes = false;
 
   // If it's a keyword type - check if it has a corresponding text type
   if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.KEYWORD) {
-    const field = newJobCapsService.getFieldById(fieldName.replace(/\.keyword$/, ''));
+    const field = newJobCapsServiceAnalytics.getFieldById(fieldName.replace(/\.keyword$/, ''));
     isBothTypes = field !== null && field.type === ES_FIELD_TYPES.TEXT;
   } else if (fieldType !== undefined && fieldType === ES_FIELD_TYPES.TEXT) {
     //   If text, check if has corresponding keyword type
-    const field = newJobCapsService.getFieldById(`${fieldName}.keyword`);
+    const field = newJobCapsServiceAnalytics.getFieldById(`${fieldName}.keyword`);
     isBothTypes = field !== null && field.type === ES_FIELD_TYPES.KEYWORD;
   }
 
@@ -180,24 +180,22 @@ export const getDefaultFieldsFromJobCaps = (
   // default is 'ml'
   const resultsField = jobConfig.dest.results_field;
 
-  const featureImportanceFields = [];
-  const topClassesFields = [];
   const allFields: any = [];
   let type: ES_FIELD_TYPES | undefined;
   let predictedField: string | undefined;
 
   if (isOutlierAnalysis(jobConfig.analysis)) {
-    if (jobConfig.analysis.outlier_detection.compute_feature_influence) {
-      featureImportanceFields.push({
-        id: `${resultsField}.${FEATURE_INFLUENCE}`,
-        name: `${resultsField}.${FEATURE_INFLUENCE}`,
-        type: KBN_FIELD_TYPES.UNKNOWN,
-      });
+    if (!jobConfig.analysis.outlier_detection.compute_feature_influence) {
+      // remove all feature influence fields
+      fields = fields.filter(
+        (field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}`)
+      );
+    } else {
+      // remove flattened feature influence fields
+      fields = fields.filter(
+        (field: any) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`)
+      );
     }
-    // remove flattened feature influence fields
-    fields = fields.filter(
-      (field) => !field.name.includes(`${resultsField}.${FEATURE_INFLUENCE}.`)
-    );
 
     // Only need to add these fields if we didn't use dest index pattern to get the fields
     if (needsDestIndexFields === true) {
@@ -211,7 +209,7 @@ export const getDefaultFieldsFromJobCaps = (
 
   if (isClassificationAnalysis(jobConfig.analysis) || isRegressionAnalysis(jobConfig.analysis)) {
     const dependentVariable = getDependentVar(jobConfig.analysis);
-    type = newJobCapsService.getFieldById(dependentVariable)?.type;
+    type = newJobCapsServiceAnalytics.getFieldById(dependentVariable)?.type;
     const predictionFieldName = getPredictionFieldName(jobConfig.analysis);
     const numTopFeatureImportanceValues = getNumTopFeatureImportanceValues(jobConfig.analysis);
     const numTopClasses = getNumTopClasses(jobConfig.analysis);
@@ -221,24 +219,24 @@ export const getDefaultFieldsFromJobCaps = (
       predictionFieldName ? predictionFieldName : defaultPredictionField
     }`;
 
-    if ((numTopFeatureImportanceValues ?? 0) > 0) {
-      featureImportanceFields.push({
-        id: `${resultsField}.${FEATURE_IMPORTANCE}`,
-        name: `${resultsField}.${FEATURE_IMPORTANCE}`,
-        type: KBN_FIELD_TYPES.UNKNOWN,
-      });
+    if ((numTopFeatureImportanceValues ?? 0) === 0) {
+      // remove all feature importance fields
+      fields = fields.filter(
+        (field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}`)
+      );
+    } else {
       // remove flattened feature importance fields
       fields = fields.filter(
         (field: any) => !field.name.includes(`${resultsField}.${FEATURE_IMPORTANCE}.`)
       );
     }
 
-    if ((numTopClasses ?? 0) > 0) {
-      topClassesFields.push({
-        id: `${resultsField}.${TOP_CLASSES}`,
-        name: `${resultsField}.${TOP_CLASSES}`,
-        type: KBN_FIELD_TYPES.UNKNOWN,
-      });
+    if ((numTopClasses ?? 0) === 0) {
+      // remove all top classes fields
+      fields = fields.filter(
+        (field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}`)
+      );
+    } else {
       // remove flattened top classes fields
       fields = fields.filter(
         (field: any) => !field.name.includes(`${resultsField}.${TOP_CLASSES}.`)
@@ -258,7 +256,7 @@ export const getDefaultFieldsFromJobCaps = (
     }
   }
 
-  allFields.push(...fields, ...featureImportanceFields, ...topClassesFields);
+  allFields.push(...fields);
   allFields.sort(({ name: a }: { name: string }, { name: b }: { name: string }) =>
     sortExplorationResultsFields(a, b, jobConfig)
   );

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/get_index_fields.ts
@@ -7,7 +7,7 @@
 
 import { ES_FIELD_TYPES } from '../../../../../../../src/plugins/data/public';
 
-import { newJobCapsService } from '../../services/new_job_capabilities_service';
+import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 
 import { getDefaultFieldsFromJobCaps, DataFrameAnalyticsConfig } from '../common';
 
@@ -19,7 +19,7 @@ export const getIndexFields = (
   jobConfig: DataFrameAnalyticsConfig | undefined,
   needsDestIndexFields: boolean
 ) => {
-  const { fields } = newJobCapsService;
+  const { fields } = newJobCapsServiceAnalytics;
   if (jobConfig !== undefined) {
     const { selectedFields: defaultSelected, docFields } = getDefaultFieldsFromJobCaps(
       fields,

diff --git a/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts b/x-pack/plugins/ml/public/application/data_frame_analytics/common/use_results_view_config.ts
@@ -15,7 +15,7 @@ import { extractErrorMessage } from '../../../../common/util/errors';
 
 import { getIndexPatternIdFromName } from '../../util/index_utils';
 import { ml } from '../../services/ml_api_service';
-import { newJobCapsService } from '../../services/new_job_capabilities_service';
+import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 import { useMlContext } from '../../contexts/ml';
 
 import { DataFrameAnalyticsConfig } from '../common';
@@ -125,7 +125,7 @@ export const useResultsViewConfig = (jobId: string) => {
             }
 
             if (indexP !== undefined) {
-              await newJobCapsService.initializeFromIndexPattern(indexP, false, false);
+              await newJobCapsServiceAnalytics.initializeFromIndexPattern(indexP);
               setJobConfig(analyticsConfigs.data_frame_analytics[0]);
               setIndexPattern(indexP);
               setIsInitialized(true);

diff --git a/...lytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx b/...lytics/pages/analytics_creation/components/configuration_step/configuration_step_form.tsx
@@ -19,7 +19,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { debounce, cloneDeep } from 'lodash';
 
-import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
+import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 import { useMlContext } from '../../../../../contexts/ml';
 import { getCombinedRuntimeMappings } from '../../../../../components/data_grid/common';
 
@@ -196,7 +196,7 @@ export const ConfigurationStepForm: FC<ConfigurationStepProps> = ({
         const depVarOptions = [];
         let depVarUpdate = formState.dependentVariable;
         // Get fields and filter for supported types for job type
-        const { fields } = newJobCapsService;
+        const { fields } = newJobCapsServiceAnalytics;
 
         let resetDependentVariable = true;
         for (const field of fields) {

diff --git a/...ytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx b/...ytics/pages/analytics_creation/components/configuration_step/supported_fields_message.tsx
@@ -16,7 +16,7 @@ import { OMIT_FIELDS } from '../../../../../../../common/constants/field_types';
 import { BASIC_NUMERICAL_TYPES, EXTENDED_NUMERICAL_TYPES } from '../../../../common/fields';
 import { CATEGORICAL_TYPES } from './form_options_validation';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../src/plugins/data/public';
-import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
+import { newJobCapsServiceAnalytics } from '../../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
 
 const containsClassificationFieldsCb = ({ name, type }: Field) =>
@@ -32,7 +32,9 @@ const containsRegressionFieldsCb = ({ name, type }: Field) =>
   (BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type));
 
 const containsOutlierFieldsCb = ({ name, type }: Field) =>
-  !OMIT_FIELDS.includes(name) && name !== EVENT_RATE_FIELD_ID && BASIC_NUMERICAL_TYPES.has(type);
+  !OMIT_FIELDS.includes(name) &&
+  name !== EVENT_RATE_FIELD_ID &&
+  (BASIC_NUMERICAL_TYPES.has(type) || EXTENDED_NUMERICAL_TYPES.has(type));
 
 const callbacks: Record<DataFrameAnalysisConfigType, (f: Field) => boolean> = {
   [ANALYSIS_CONFIG_TYPE.CLASSIFICATION]: containsClassificationFieldsCb,
@@ -71,7 +73,7 @@ export const SupportedFieldsMessage: FC<Props> = ({ jobType }) => {
     setSourceIndexContainsSupportedFields,
   ] = useState<boolean>(true);
   const [sourceIndexFieldsCheckFailed, setSourceIndexFieldsCheckFailed] = useState<boolean>(false);
-  const { fields } = newJobCapsService;
+  const { fields } = newJobCapsServiceAnalytics;
 
   // Find out if index pattern contains supported fields for job type. Provides a hint in the form
   // that job may not run correctly if no supported fields are found.
@@ -90,8 +92,6 @@ export const SupportedFieldsMessage: FC<Props> = ({ jobType }) => {
 
   useEffect(() => {
     if (jobType !== undefined) {
-      setSourceIndexContainsSupportedFields(true);
-      setSourceIndexFieldsCheckFailed(false);
       validateFields();
     }
   }, [jobType]);

diff --git a/.../public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts b/.../public/application/data_frame_analytics/pages/analytics_creation/hooks/use_index_data.ts
@@ -15,6 +15,7 @@ import { IndexPattern } from '../../../../../../../../../src/plugins/data/public
 import { isRuntimeMappings } from '../../../../../../common/util/runtime_field_utils';
 import { RuntimeMappings } from '../../../../../../common/types/fields';
 import { DEFAULT_SAMPLER_SHARD_SIZE } from '../../../../../../common/constants/field_histograms';
+import { newJobCapsServiceAnalytics } from '../../../../services/new_job_capabilities/new_job_capabilities_service_analytics';
 
 import { DataLoader } from '../../../../datavisualizer/index_based/data_loader';
 
@@ -49,6 +50,41 @@ function getRuntimeFieldColumns(runtimeMappings: RuntimeMappings) {
   });
 }
 
+function getInitialColumns(indexPattern: IndexPattern) {
+  const { fields } = newJobCapsServiceAnalytics;
+  const columns = fields.map((field: any) => {
+    const schema =
+      getDataGridSchemaFromESFieldType(field.type) || getDataGridSchemaFromKibanaFieldType(field);
+
+    return {
+      id: field.name,
+      schema,
+      isExpandable: schema !== 'boolean',
+      isRuntimeFieldColumn: false,
+    };
+  });
+
+  // Add runtime fields defined in index pattern to columns
+  if (indexPattern) {
+    const computedFields = indexPattern?.getComputedFields();
+
+    if (isRuntimeMappings(computedFields.runtimeFields)) {
+      Object.keys(computedFields.runtimeFields).forEach((runtimeField) => {
+        const schema = getDataGridSchemaFromESFieldType(
+          computedFields.runtimeFields[runtimeField].type
+        );
+        columns.push({
+          id: runtimeField,
+          schema,
+          isExpandable: schema !== 'boolean',
+          isRuntimeFieldColumn: true,
+        });
+      });
+    }
+  }
+  return columns;
+}
+
 export const useIndexData = (
   indexPattern: IndexPattern,
   query: Record<string, any> | undefined,
@@ -58,23 +94,7 @@ export const useIndexData = (
   const indexPatternFields = useMemo(() => getFieldsFromKibanaIndexPattern(indexPattern), [
     indexPattern,
   ]);
-
-  const [columns, setColumns] = useState<MLEuiDataGridColumn[]>([
-    ...indexPatternFields.map((id) => {
-      const field = indexPattern.fields.getByName(id);
-      const isRuntimeFieldColumn = field?.runtimeField !== undefined;
-      const schema = isRuntimeFieldColumn
-        ? getDataGridSchemaFromESFieldType(field?.type as estypes.RuntimeField['type'])
-        : getDataGridSchemaFromKibanaFieldType(field);
-      return {
-        id,
-        schema,
-        isExpandable: schema !== 'boolean',
-        isRuntimeFieldColumn,
-      };
-    }),
-  ]);
-
+  const [columns, setColumns] = useState<MLEuiDataGridColumn[]>(getInitialColumns(indexPattern));
   const dataGrid = useDataGrid(columns);
 
   const {
@@ -131,18 +151,7 @@ export const useIndexData = (
           ...(combinedRuntimeMappings ? getRuntimeFieldColumns(combinedRuntimeMappings) : []),
         ]);
       } else {
-        setColumns([
-          ...indexPatternFields.map((id) => {
-            const field = indexPattern.fields.getByName(id);
-            const schema = getDataGridSchemaFromKibanaFieldType(field);
-            return {
-              id,
-              schema,
-              isExpandable: schema !== 'boolean',
-              isRuntimeFieldColumn: field?.runtimeField !== undefined,
-            };
-          }),
-        ]);
+        setColumns(getInitialColumns(indexPattern));
       }
       setRowCount(typeof resp.hits.total === 'number' ? resp.hits.total : resp.hits.total.value);
       setRowCountRelation(

diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts
@@ -8,7 +8,7 @@
 import { i18n } from '@kbn/i18n';
 
 import { Job, Datafeed, Detector } from '../../../../../../../common/types/anomaly_detection_jobs';
-import { newJobCapsService } from '../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import { NavigateToPath } from '../../../../../contexts/kibana';
 import {
   ML_JOB_AGGREGATION,

diff --git a/...lication/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx b/...lication/jobs/new_job/pages/components/datafeed_step/components/time_field/time_field.tsx
@@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
 
 import { TimeFieldSelect } from './time_field_select';
 import { JobCreatorContext } from '../../../job_creator_context';
-import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import { AdvancedJobCreator } from '../../../../../common/job_creator';
 import { Description } from './description';
 

diff --git a/...s/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx b/...s/new_job/pages/components/pick_fields_step/components/advanced_view/metric_selection.tsx
@@ -9,7 +9,7 @@ import React, { Fragment, FC, useContext, useState } from 'react';
 
 import { JobCreatorContext } from '../../../job_creator_context';
 import { AdvancedJobCreator } from '../../../../../common/job_creator';
-import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import { Aggregation, Field } from '../../../../../../../../../common/types/fields';
 import { MetricSelector } from './metric_selector';
 import { RichDetector } from '../../../../../common/job_creator/advanced_job_creator';

diff --git a/...ages/components/pick_fields_step/components/categorization_field/categorization_field.tsx b/...ages/components/pick_fields_step/components/categorization_field/categorization_field.tsx
@@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
 
 import { CategorizationFieldSelect } from './categorization_field_select';
 import { JobCreatorContext } from '../../../job_creator_context';
-import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import {
   AdvancedJobCreator,
   CategorizationJobCreator,

diff --git a/..._step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx b/..._step/components/categorization_partition_field/categorization_per_partition_dropdown.tsx
@@ -10,7 +10,7 @@ import { EuiFormRow } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { JobCreatorContext } from '../../../job_creator_context';
 import { CategorizationJobCreator } from '../../../../../common/job_creator';
-import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import { CategorizationPerPartitionFieldSelect } from './categorization_per_partition_input';
 
 export const CategorizationPerPartitionFieldDropdown = ({

diff --git a/...ion/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx b/...ion/jobs/new_job/pages/components/pick_fields_step/components/influencers/influencers.tsx
@@ -9,7 +9,7 @@ import React, { FC, useContext, useEffect, useState } from 'react';
 
 import { InfluencersSelect } from './influencers_select';
 import { JobCreatorContext } from '../../../job_creator_context';
-import { newJobCapsService } from '../../../../../../../services/new_job_capabilities_service';
+import { newJobCapsService } from '../../../../../../../services/new_job_capabilities/new_job_capabilities_service';
 import {
   MultiMetricJobCreator,
   PopulationJobCreator,