From 641e03bda1383cfb5831e9b7e5591e57b0c6fcc8 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Thu, 19 Mar 2020 16:52:27 +0100 Subject: [PATCH 01/15] [ML] add estimateModelMemory to the setup endpoint --- .../jobs/new_job/recognize/page.tsx | 1 + .../services/ml_api_service/index.ts | 3 + .../models/data_recognizer/data_recognizer.ts | 148 ++++++++++++++---- .../models/fields_service/fields_service.ts | 67 ++++---- x-pack/plugins/ml/server/routes/modules.ts | 36 ++--- .../ml/server/routes/schemas/modules.ts | 5 + 6 files changed, 178 insertions(+), 82 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx index 50c35ec426acb..c5192f99ce48a 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx @@ -172,6 +172,7 @@ export const Page: FC = ({ moduleId, existingGroupIds }) => { startDatafeed: startDatafeedAfterSave, ...(jobOverridesPayload !== null ? { jobOverrides: jobOverridesPayload } : {}), ...resultTimeRange, + estimateModelMemory: true, }); const { datafeeds: datafeedsResponse, jobs: jobsResponse, kibana: kibanaResponse } = response; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index cd4a97bd10ed4..df59678452e2f 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -367,6 +367,7 @@ export const ml = { start, end, jobOverrides, + estimateModelMemory, }: { moduleId: string; prefix?: string; @@ -378,6 +379,7 @@ export const ml = { start?: number; end?: number; jobOverrides?: Array>; + estimateModelMemory?: boolean; }) { const body = JSON.stringify({ prefix, @@ -389,6 +391,7 @@ export const ml = { start, end, jobOverrides, + estimateModelMemory, }); return http({ diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index a54c2f22a7951..320bea778998c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -10,7 +10,7 @@ import numeral from '@elastic/numeral'; import { CallAPIOptions, APICaller, SavedObjectsClientContract } from 'kibana/server'; import { IndexPatternAttributes } from 'src/plugins/data/server'; import { merge } from 'lodash'; -import { CombinedJobWithStats } from '../../../common/types/anomaly_detection_jobs'; +import { AnalysisLimits, CombinedJobWithStats } from '../../../common/types/anomaly_detection_jobs'; import { KibanaObjects, ModuleDataFeed, @@ -26,6 +26,8 @@ import { } from '../../../common/types/modules'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; +import { calculateModelMemoryLimitProvider } from '../calculate_model_memory_limit'; +import { fieldsServiceProvider } from '../fields_service'; import { jobServiceProvider } from '../job_service'; import { resultsServiceProvider } from '../results_service'; @@ -367,16 +369,17 @@ export class DataRecognizer { // if any of the savedObjects already exist, they will not be overwritten. async setupModuleItems( moduleId: string, - jobPrefix: string, - groups: string[], - indexPatternName: string, + jobPrefix: string | undefined, + groups: string[] | undefined, + indexPatternName: string | undefined, query: any, - useDedicatedIndex: boolean, - startDatafeed: boolean, - start: number, - end: number, + useDedicatedIndex: boolean | undefined, + startDatafeed: boolean | undefined, + start: number | undefined, + end: number | undefined, jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[] + datafeedOverrides: DatafeedOverride[], + estimateModelMemory: boolean | undefined ) { // load the config from disk const moduleConfig = await this.getModule(moduleId, jobPrefix); @@ -422,7 +425,7 @@ export class DataRecognizer { this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix); this.updateDatafeedIndices(moduleConfig); this.updateJobUrlIndexPatterns(moduleConfig); - await this.updateModelMemoryLimits(moduleConfig); + await this.updateModelMemoryLimits(moduleConfig, estimateModelMemory, start, end); // create the jobs if (moduleConfig.jobs && moduleConfig.jobs.length) { @@ -689,8 +692,8 @@ export class DataRecognizer { async startDatafeeds( datafeeds: ModuleDataFeed[], - start: number, - end: number + start?: number, + end?: number ): Promise<{ [key: string]: DatafeedResponse }> { const results = {} as { [key: string]: DatafeedResponse }; for (const datafeed of datafeeds) { @@ -933,28 +936,111 @@ export class DataRecognizer { } } - // ensure the model memory limit for each job is not greater than - // the max model memory setting for the cluster - async updateModelMemoryLimits(moduleConfig: Module) { + /** + * Checks if the time range is provided and fallbacks + * to the last 3 month of data + */ + async getFallbackTimeRange( + timeField: string, + query?: any + ): Promise<{ start: number; end: number }> { + const fieldsService = fieldsServiceProvider(this.callAsCurrentUser); + const timeFieldRange = await fieldsService.getTimeFieldRange( + this.indexPatternName, + timeField, + query + ); + return { + start: timeFieldRange.start.epoch, + end: timeFieldRange.end.epoch, + }; + } + + /** + * Ensure the model memory limit for each job is not greater than + * the max model memory setting for the cluster + */ + async updateModelMemoryLimits( + moduleConfig: Module, + estimateMML: boolean = false, + start?: number, + end?: number + ) { const { limits } = await this.callAsCurrentUser('ml.info'); + const maxMml = limits.max_model_memory_limit; - if (maxMml !== undefined) { - // @ts-ignore - const maxBytes: number = numeral(maxMml.toUpperCase()).value(); - - if (Array.isArray(moduleConfig.jobs)) { - moduleConfig.jobs.forEach(job => { - const mml = job.config?.analysis_limits?.model_memory_limit; - if (mml !== undefined) { - // @ts-ignore - const mmlBytes: number = numeral(mml.toUpperCase()).value(); - if (mmlBytes > maxBytes) { - // if the job's mml is over the max, - // so set the jobs mml to be the max - job.config.analysis_limits!.model_memory_limit = maxMml; - } + // @ts-ignore + const maxBytes: number = numeral(maxMml.toUpperCase()).value(); + + if (!Array.isArray(moduleConfig.jobs)) { + return; + } + + if (estimateMML) { + const calculateModelMemoryLimit = calculateModelMemoryLimitProvider(this.callAsCurrentUser); + const query = moduleConfig.query ?? null; + + const isSameTimeFields = moduleConfig.jobs.every( + job => + job.config.data_description.time_field === + moduleConfig.jobs[0].config.data_description.time_field + ); + + if (isSameTimeFields && (start === undefined || end === undefined)) { + const { start: fallbackStart, end: fallbackEnd } = await this.getFallbackTimeRange( + moduleConfig.jobs[0].config.data_description.time_field, + query + ); + start = fallbackStart; + end = fallbackEnd; + } + + for (const job of moduleConfig.jobs) { + let earliestMs = start; + let latestMs = end; + if (earliestMs === undefined || latestMs === undefined) { + const timeFieldRange = await this.getFallbackTimeRange( + job.config.data_description.time_field, + query + ); + earliestMs = timeFieldRange.start; + latestMs = timeFieldRange.end; + } + + const { modelMemoryLimit } = await calculateModelMemoryLimit( + job.config.analysis_config, + this.indexPatternName, + query, + job.config.data_description.time_field, + earliestMs, + latestMs + ); + + if (!job.config.analysis_limits) { + job.config.analysis_limits = {} as AnalysisLimits; + } + + job.config.analysis_limits.model_memory_limit = modelMemoryLimit; + } + + return; + } + + for (const job of moduleConfig.jobs) { + const mml = job.config?.analysis_limits?.model_memory_limit; + if (mml !== undefined) { + // @ts-ignore + const mmlBytes: number = numeral(mml.toUpperCase()).value(); + if (mmlBytes > maxBytes) { + // if the job's mml is over the max, + // so set the jobs mml to be the max + + if (!job.config.analysis_limits) { + job.config.analysis_limits = {} as AnalysisLimits; } - }); + + job.config.analysis_limits.model_memory_limit = maxMml; + } } } } diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index d16984abc5d2a..aae79706c5588 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -111,47 +111,48 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { }, {} as { [field: string]: number }); } - function getTimeFieldRange( + /** + * Gets time boundaries of the index data based on the provided time field. + */ + async function getTimeFieldRange( index: string[] | string, timeFieldName: string, query: any - ): Promise { - return new Promise((resolve, reject) => { - const obj = { success: true, start: { epoch: 0, string: '' }, end: { epoch: 0, string: '' } }; - - callAsCurrentUser('search', { - index, - size: 0, - body: { - query, - aggs: { - earliest: { - min: { - field: timeFieldName, - }, + ): Promise<{ + success: boolean; + start: { epoch: number; string: string }; + end: { epoch: number; string: string }; + }> { + const obj = { success: true, start: { epoch: 0, string: '' }, end: { epoch: 0, string: '' } }; + + const resp = await callAsCurrentUser('search', { + index, + size: 0, + body: { + ...(query ? { query } : {}), + aggs: { + earliest: { + min: { + field: timeFieldName, }, - latest: { - max: { - field: timeFieldName, - }, + }, + latest: { + max: { + field: timeFieldName, }, }, }, - }) - .then(resp => { - if (resp.aggregations && resp.aggregations.earliest && resp.aggregations.latest) { - obj.start.epoch = resp.aggregations.earliest.value; - obj.start.string = resp.aggregations.earliest.value_as_string; - - obj.end.epoch = resp.aggregations.latest.value; - obj.end.string = resp.aggregations.latest.value_as_string; - } - resolve(obj); - }) - .catch(resp => { - reject(resp); - }); + }, }); + + if (resp.aggregations && resp.aggregations.earliest && resp.aggregations.latest) { + obj.start.epoch = resp.aggregations.earliest.value; + obj.start.string = resp.aggregations.earliest.value_as_string; + + obj.end.epoch = resp.aggregations.latest.value; + obj.end.string = resp.aggregations.latest.value_as_string; + } + return obj; } /** diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index 685119672a983..797f943031625 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { schema } from '@kbn/config-schema'; +import { schema, TypeOf } from '@kbn/config-schema'; import { RequestHandlerContext } from 'kibana/server'; import { DatafeedOverride, JobOverride } from '../../common/types/modules'; @@ -36,16 +36,17 @@ function getModule(context: RequestHandlerContext, moduleId: string) { function saveModuleItems( context: RequestHandlerContext, moduleId: string, - prefix: string, - groups: string[], - indexPatternName: string, + prefix: string | undefined, + groups: string[] | undefined, + indexPatternName: string | undefined, query: any, - useDedicatedIndex: boolean, - startDatafeed: boolean, - start: number, - end: number, + useDedicatedIndex: boolean | undefined, + startDatafeed: boolean | undefined, + start: number | undefined, + end: number | undefined, jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[] + datafeedOverrides: DatafeedOverride[], + estimateModelMemory: boolean | undefined ) { const dr = new DataRecognizer( context.ml!.mlClient.callAsCurrentUser, @@ -62,7 +63,8 @@ function saveModuleItems( start, end, jobOverrides, - datafeedOverrides + datafeedOverrides, + estimateModelMemory ); } @@ -156,9 +158,7 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { { path: '/api/ml/modules/setup/{moduleId}', validate: { - params: schema.object({ - ...getModuleIdParamSchema(), - }), + params: schema.object(getModuleIdParamSchema()), body: setupModuleBodySchema, }, }, @@ -177,7 +177,8 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { end, jobOverrides, datafeedOverrides, - } = request.body; + estimateModelMemory, + } = request.body as TypeOf; const result = await saveModuleItems( context, @@ -191,7 +192,8 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { start, end, jobOverrides, - datafeedOverrides + datafeedOverrides, + estimateModelMemory ); return response.ok({ body: result }); @@ -214,9 +216,7 @@ export function dataRecognizer({ router, mlLicense }: RouteInitialization) { { path: '/api/ml/modules/jobs_exist/{moduleId}', validate: { - params: schema.object({ - ...getModuleIdParamSchema(), - }), + params: schema.object(getModuleIdParamSchema()), }, }, mlLicense.fullLicenseAPIGuard(async (context, request, response) => { diff --git a/x-pack/plugins/ml/server/routes/schemas/modules.ts b/x-pack/plugins/ml/server/routes/schemas/modules.ts index 46b7e53c22a05..98e3d80f0ff84 100644 --- a/x-pack/plugins/ml/server/routes/schemas/modules.ts +++ b/x-pack/plugins/ml/server/routes/schemas/modules.ts @@ -17,6 +17,11 @@ export const setupModuleBodySchema = schema.object({ end: schema.maybe(schema.number()), jobOverrides: schema.maybe(schema.any()), datafeedOverrides: schema.maybe(schema.any()), + /** + * Indicates whether an estimate of the model memory limit + * should be made by checking the cardinality of fields in the job configurations. + */ + estimateModelMemory: schema.maybe(schema.boolean()), }); export const getModuleIdParamSchema = (optional = false) => { From 370f4db5e1597fb9ddc132b336e432cd3475942c Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 20 Mar 2020 13:51:01 +0100 Subject: [PATCH 02/15] [ML] wip caching cardinality checks --- .../calculate_model_memory_limit.ts | 238 ++++++++++++------ 1 file changed, 164 insertions(+), 74 deletions(-) diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index c97bbe07fffda..1715aaeaee37f 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -5,6 +5,7 @@ */ import numeral from '@elastic/numeral'; +import { pick } from 'lodash'; import { APICaller } from 'kibana/server'; import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs'; import { fieldsServiceProvider } from '../fields_service'; @@ -34,92 +35,182 @@ export interface ModelMemoryEstimate { /** * Retrieves overall and max bucket cardinalities. */ -async function getCardinalities( - callAsCurrentUser: APICaller, - analysisConfig: AnalysisConfig, - indexPattern: string, - query: any, - timeFieldName: string, - earliestMs: number, - latestMs: number -): Promise<{ - overallCardinality: { [key: string]: number }; - maxBucketCardinality: { [key: string]: number }; -}> { - /** - * Fields not involved in cardinality check - */ - const excludedKeywords = new Set( +const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { + const cardinalityCache = new Map< + string, + { + overallCardinality: { [field: string]: number }; + maxBucketCardinality: { [field: string]: number }; + } + >(); + + const updateCache = ( + indexPattern: string, + timeField: string, + earliestMs: number, + latestMs: number, + update: { + overallCardinality?: { [field: string]: number }; + maxBucketCardinality?: { [field: string]: number }; + } + ): void => { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cachedValues = cardinalityCache.get(cacheKey); + if (cachedValues === undefined) { + cardinalityCache.set(cacheKey, { + overallCardinality: update.overallCardinality ?? {}, + maxBucketCardinality: update.maxBucketCardinality ?? {}, + }); + return; + } + + Object.assign(cachedValues.overallCardinality, update.overallCardinality); + Object.assign(cachedValues.maxBucketCardinality, update.maxBucketCardinality); + }; + + const getCacheValue = ( + indexPattern: string, + timeField: string, + earliestMs: number, + latestMs: number, + overallCardinalityFields: string[], + maxBucketCardinalityFields: string[] + ): { + overallCardinality: { [field: string]: number }; + maxBucketCardinality: { [field: string]: number }; + } | null => { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cached = cardinalityCache.get(cacheKey); + if (!cached) { + return null; + } + + const overallCardinality = pick(cached.overallCardinality, overallCardinalityFields); + const maxBucketCardinality = pick(cached.maxBucketCardinality, maxBucketCardinalityFields); + + return { + overallCardinality, + maxBucketCardinality, + }; + }; + + return async ( + analysisConfig: AnalysisConfig, + indexPattern: string, + query: any, + timeFieldName: string, + earliestMs: number, + latestMs: number + ): Promise<{ + overallCardinality: { [key: string]: number }; + maxBucketCardinality: { [key: string]: number }; + }> => { /** - * The keyword which is used to mean the output of categorization, - * so it will have cardinality zero in the actual input data. + * Fields not involved in cardinality check */ - 'mlcategory' - ); - - const fieldsService = fieldsServiceProvider(callAsCurrentUser); - - const { detectors, influencers, bucket_span: bucketSpan } = analysisConfig; - - let overallCardinality = {}; - let maxBucketCardinality = {}; - const overallCardinalityFields: Set = detectors.reduce( - ( - acc, - { - by_field_name: byFieldName, - partition_field_name: partitionFieldName, - over_field_name: overFieldName, - } - ) => { - [byFieldName, partitionFieldName, overFieldName] - .filter(field => field !== undefined && field !== '' && !excludedKeywords.has(field)) - .forEach(key => { - acc.add(key as string); - }); - return acc; - }, - new Set() - ); - - const maxBucketFieldCardinalities: string[] = influencers.filter( - influencerField => - typeof influencerField === 'string' && - !excludedKeywords.has(influencerField) && - !!influencerField && - !overallCardinalityFields.has(influencerField) - ) as string[]; - - if (overallCardinalityFields.size > 0) { - overallCardinality = await fieldsService.getCardinalityOfFields( - indexPattern, - [...overallCardinalityFields], - query, - timeFieldName, - earliestMs, - latestMs + const excludedKeywords = new Set( + /** + * The keyword which is used to mean the output of categorization, + * so it will have cardinality zero in the actual input data. + */ + 'mlcategory' + ); + + const fieldsService = fieldsServiceProvider(callAsCurrentUser); + + const { detectors, influencers, bucket_span: bucketSpan } = analysisConfig; + + let overallCardinality = {}; + let maxBucketCardinality = {}; + + // Get fields required for the model memory estimation + const overallCardinalityFields: Set = detectors.reduce( + ( + acc, + { + by_field_name: byFieldName, + partition_field_name: partitionFieldName, + over_field_name: overFieldName, + } + ) => { + [byFieldName, partitionFieldName, overFieldName] + .filter(field => field !== undefined && field !== '' && !excludedKeywords.has(field)) + .forEach(key => { + acc.add(key as string); + }); + return acc; + }, + new Set() ); - } - if (maxBucketFieldCardinalities.length > 0) { - maxBucketCardinality = await fieldsService.getMaxBucketCardinalities( + const maxBucketFieldCardinalities: string[] = influencers.filter( + influencerField => + !!influencerField && + !excludedKeywords.has(influencerField) && + !overallCardinalityFields.has(influencerField) + ) as string[]; + + // Check if some of the values are already cached + const cachedValues = getCacheValue( indexPattern, - maxBucketFieldCardinalities, - query, timeFieldName, earliestMs, latestMs, - bucketSpan + [...overallCardinalityFields], + maxBucketFieldCardinalities ); - } + overallCardinality = cachedValues?.overallCardinality ?? {}; + maxBucketCardinality = cachedValues?.maxBucketCardinality ?? {}; - return { - overallCardinality, - maxBucketCardinality, + const overallCardinalityFieldsToFetch = [...overallCardinalityFields].filter( + v => !overallCardinality.hasOwnProperty(v) + ); + if (overallCardinalityFieldsToFetch.length > 0) { + const overallCardinalitResp = await fieldsService.getCardinalityOfFields( + indexPattern, + overallCardinalityFieldsToFetch, + query, + timeFieldName, + earliestMs, + latestMs + ); + overallCardinality = { ...overallCardinality, ...overallCardinalitResp }; + // update cache + updateCache(indexPattern, timeFieldName, earliestMs, latestMs, { + overallCardinality: overallCardinalitResp, + }); + } + + const maxBucketCardinalityFeildToFetch = maxBucketFieldCardinalities.filter( + v => !maxBucketCardinality.hasOwnProperty(v) + ); + if (maxBucketCardinalityFeildToFetch.length > 0) { + const maxBucketCardinalityResp = await fieldsService.getMaxBucketCardinalities( + indexPattern, + maxBucketCardinalityFeildToFetch, + query, + timeFieldName, + earliestMs, + latestMs, + bucketSpan + ); + maxBucketCardinality = { ...maxBucketCardinality, ...maxBucketCardinalityResp }; + // update cache + updateCache(indexPattern, timeFieldName, earliestMs, latestMs, { + maxBucketCardinality: maxBucketCardinalityResp, + }); + } + + return { + overallCardinality, + maxBucketCardinality, + }; }; -} +}; export function calculateModelMemoryLimitProvider(callAsCurrentUser: APICaller) { + const getCardinalities = cardinalityCheckProvider(callAsCurrentUser); + /** * Retrieves an estimated size of the model memory limit used in the job config * based on the cardinality of the fields being used to split the data @@ -145,7 +236,6 @@ export function calculateModelMemoryLimitProvider(callAsCurrentUser: APICaller) } const { overallCardinality, maxBucketCardinality } = await getCardinalities( - callAsCurrentUser, analysisConfig, indexPattern, query, From 35f441e6f24e5da7ec1d0c607725bf1bc5e7377b Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 20 Mar 2020 14:03:39 +0100 Subject: [PATCH 03/15] [ML] refactor --- .../calculate_model_memory_limit.ts | 125 ++++++++++-------- 1 file changed, 73 insertions(+), 52 deletions(-) diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index 1715aaeaee37f..fb6ff82d8e3c0 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -33,9 +33,10 @@ export interface ModelMemoryEstimate { } /** - * Retrieves overall and max bucket cardinalities. + * Caches cardinality fields values to avoid + * unnecessary aggregations on elasticsearch */ -const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { +const initCardinalityFieldsCache = () => { const cardinalityCache = new Map< string, { @@ -44,55 +45,75 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { } >(); - const updateCache = ( - indexPattern: string, - timeField: string, - earliestMs: number, - latestMs: number, - update: { - overallCardinality?: { [field: string]: number }; - maxBucketCardinality?: { [field: string]: number }; - } - ): void => { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; - const cachedValues = cardinalityCache.get(cacheKey); - if (cachedValues === undefined) { - cardinalityCache.set(cacheKey, { - overallCardinality: update.overallCardinality ?? {}, - maxBucketCardinality: update.maxBucketCardinality ?? {}, - }); - return; - } + return { + /** + * Gets requested values from cache + * @param indexPattern + * @param timeField + * @param earliestMs + * @param latestMs + * @param overallCardinalityFields - array of overall cardinality fields to retrieve form cache + * @param maxBucketCardinalityFields - array of max bucket cardinality field to retrieve from cache + */ + getValues( + indexPattern: string, + timeField: string, + earliestMs: number, + latestMs: number, + overallCardinalityFields: string[], + maxBucketCardinalityFields: string[] + ): { + overallCardinality: { [field: string]: number }; + maxBucketCardinality: { [field: string]: number }; + } | null { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cached = cardinalityCache.get(cacheKey); + if (!cached) { + return null; + } - Object.assign(cachedValues.overallCardinality, update.overallCardinality); - Object.assign(cachedValues.maxBucketCardinality, update.maxBucketCardinality); - }; + const overallCardinality = pick(cached.overallCardinality, overallCardinalityFields); + const maxBucketCardinality = pick(cached.maxBucketCardinality, maxBucketCardinalityFields); - const getCacheValue = ( - indexPattern: string, - timeField: string, - earliestMs: number, - latestMs: number, - overallCardinalityFields: string[], - maxBucketCardinalityFields: string[] - ): { - overallCardinality: { [field: string]: number }; - maxBucketCardinality: { [field: string]: number }; - } | null => { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; - const cached = cardinalityCache.get(cacheKey); - if (!cached) { - return null; - } - - const overallCardinality = pick(cached.overallCardinality, overallCardinalityFields); - const maxBucketCardinality = pick(cached.maxBucketCardinality, maxBucketCardinalityFields); + return { + overallCardinality, + maxBucketCardinality, + }; + }, + /** + * Extends cache with provided values + */ + updateValues( + indexPattern: string, + timeField: string, + earliestMs: number, + latestMs: number, + update: { + overallCardinality?: { [field: string]: number }; + maxBucketCardinality?: { [field: string]: number }; + } + ): void { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cachedValues = cardinalityCache.get(cacheKey); + if (cachedValues === undefined) { + cardinalityCache.set(cacheKey, { + overallCardinality: update.overallCardinality ?? {}, + maxBucketCardinality: update.maxBucketCardinality ?? {}, + }); + return; + } - return { - overallCardinality, - maxBucketCardinality, - }; + Object.assign(cachedValues.overallCardinality, update.overallCardinality); + Object.assign(cachedValues.maxBucketCardinality, update.maxBucketCardinality); + }, }; +}; + +/** + * Retrieves overall and max bucket cardinalities. + */ +const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { + const cardinalityFieldsCache = initCardinalityFieldsCache(); return async ( analysisConfig: AnalysisConfig, @@ -151,7 +172,7 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { ) as string[]; // Check if some of the values are already cached - const cachedValues = getCacheValue( + const cachedValues = cardinalityFieldsCache.getValues( indexPattern, timeFieldName, earliestMs, @@ -175,8 +196,8 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { latestMs ); overallCardinality = { ...overallCardinality, ...overallCardinalitResp }; - // update cache - updateCache(indexPattern, timeFieldName, earliestMs, latestMs, { + + cardinalityFieldsCache.updateValues(indexPattern, timeFieldName, earliestMs, latestMs, { overallCardinality: overallCardinalitResp, }); } @@ -195,8 +216,8 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { bucketSpan ); maxBucketCardinality = { ...maxBucketCardinality, ...maxBucketCardinalityResp }; - // update cache - updateCache(indexPattern, timeFieldName, earliestMs, latestMs, { + + cardinalityFieldsCache.updateValues(indexPattern, timeFieldName, earliestMs, latestMs, { maxBucketCardinality: maxBucketCardinalityResp, }); } From c67dc3a7b9b5d6925ae6568f0fcae7ff960275b9 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 20 Mar 2020 14:16:38 +0100 Subject: [PATCH 04/15] [ML] fix a fallback time range --- .../ml/server/models/data_recognizer/data_recognizer.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 320bea778998c..c035ca74c3682 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -8,6 +8,7 @@ import fs from 'fs'; import Boom from 'boom'; import numeral from '@elastic/numeral'; import { CallAPIOptions, APICaller, SavedObjectsClientContract } from 'kibana/server'; +import moment from 'moment'; import { IndexPatternAttributes } from 'src/plugins/data/server'; import { merge } from 'lodash'; import { AnalysisLimits, CombinedJobWithStats } from '../../../common/types/anomaly_detection_jobs'; @@ -937,21 +938,22 @@ export class DataRecognizer { } /** - * Checks if the time range is provided and fallbacks - * to the last 3 month of data + * Provides a time range of the last 3 months of data */ async getFallbackTimeRange( timeField: string, query?: any ): Promise<{ start: number; end: number }> { const fieldsService = fieldsServiceProvider(this.callAsCurrentUser); + const timeFieldRange = await fieldsService.getTimeFieldRange( this.indexPatternName, timeField, query ); + return { - start: timeFieldRange.start.epoch, + start: timeFieldRange.end.epoch - moment.duration(3, 'months').asMilliseconds(), end: timeFieldRange.end.epoch, }; } From a46aa5335a158df7e5caef95b828662759309919 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 20 Mar 2020 16:26:26 +0100 Subject: [PATCH 05/15] [ML] fix typing issue --- .../ml/server/models/data_recognizer/data_recognizer.ts | 2 +- .../ml/server/shared_services/providers/modules.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index c035ca74c3682..9c4dfa81af783 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -380,7 +380,7 @@ export class DataRecognizer { end: number | undefined, jobOverrides: JobOverride[], datafeedOverrides: DatafeedOverride[], - estimateModelMemory: boolean | undefined + estimateModelMemory?: boolean ) { // load the config from disk const moduleConfig = await this.getModule(moduleId, jobPrefix); diff --git a/x-pack/plugins/ml/server/shared_services/providers/modules.ts b/x-pack/plugins/ml/server/shared_services/providers/modules.ts index ffc977917ae46..ec876273c2c33 100644 --- a/x-pack/plugins/ml/server/shared_services/providers/modules.ts +++ b/x-pack/plugins/ml/server/shared_services/providers/modules.ts @@ -32,7 +32,8 @@ export interface ModulesProvider { start: number, end: number, jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[] + datafeedOverrides: DatafeedOverride[], + estimateModelMemory?: boolean ): Promise; }; } @@ -65,7 +66,8 @@ export function getModulesProvider(isFullLicense: LicenseCheck): ModulesProvider start: number, end: number, jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[] + datafeedOverrides: DatafeedOverride[], + estimateModelMemory?: boolean ) { const dr = dataRecognizerFactory(callAsCurrentUser, savedObjectsClient); return dr.setupModuleItems( @@ -79,7 +81,8 @@ export function getModulesProvider(isFullLicense: LicenseCheck): ModulesProvider start, end, jobOverrides, - datafeedOverrides + datafeedOverrides, + estimateModelMemory ); }, }; From cbc5f0d48be25681eeb855ec02bf20150ef85ba7 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Sun, 22 Mar 2020 17:35:16 +0100 Subject: [PATCH 06/15] [ML] fields_aggs_cache.ts as part of fields_service --- .../calculate_model_memory_limit.ts | 122 +----------------- .../fields_service/fields_aggs_cache.ts | 66 ++++++++++ .../models/fields_service/fields_service.ts | 61 ++++++++- 3 files changed, 129 insertions(+), 120 deletions(-) create mode 100644 x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index fb6ff82d8e3c0..70bbc110ac050 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -5,7 +5,6 @@ */ import numeral from '@elastic/numeral'; -import { pick } from 'lodash'; import { APICaller } from 'kibana/server'; import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs'; import { fieldsServiceProvider } from '../fields_service'; @@ -32,88 +31,11 @@ export interface ModelMemoryEstimate { model_memory_estimate: string; } -/** - * Caches cardinality fields values to avoid - * unnecessary aggregations on elasticsearch - */ -const initCardinalityFieldsCache = () => { - const cardinalityCache = new Map< - string, - { - overallCardinality: { [field: string]: number }; - maxBucketCardinality: { [field: string]: number }; - } - >(); - - return { - /** - * Gets requested values from cache - * @param indexPattern - * @param timeField - * @param earliestMs - * @param latestMs - * @param overallCardinalityFields - array of overall cardinality fields to retrieve form cache - * @param maxBucketCardinalityFields - array of max bucket cardinality field to retrieve from cache - */ - getValues( - indexPattern: string, - timeField: string, - earliestMs: number, - latestMs: number, - overallCardinalityFields: string[], - maxBucketCardinalityFields: string[] - ): { - overallCardinality: { [field: string]: number }; - maxBucketCardinality: { [field: string]: number }; - } | null { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; - const cached = cardinalityCache.get(cacheKey); - if (!cached) { - return null; - } - - const overallCardinality = pick(cached.overallCardinality, overallCardinalityFields); - const maxBucketCardinality = pick(cached.maxBucketCardinality, maxBucketCardinalityFields); - - return { - overallCardinality, - maxBucketCardinality, - }; - }, - /** - * Extends cache with provided values - */ - updateValues( - indexPattern: string, - timeField: string, - earliestMs: number, - latestMs: number, - update: { - overallCardinality?: { [field: string]: number }; - maxBucketCardinality?: { [field: string]: number }; - } - ): void { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; - const cachedValues = cardinalityCache.get(cacheKey); - if (cachedValues === undefined) { - cardinalityCache.set(cacheKey, { - overallCardinality: update.overallCardinality ?? {}, - maxBucketCardinality: update.maxBucketCardinality ?? {}, - }); - return; - } - - Object.assign(cachedValues.overallCardinality, update.overallCardinality); - Object.assign(cachedValues.maxBucketCardinality, update.maxBucketCardinality); - }, - }; -}; - /** * Retrieves overall and max bucket cardinalities. */ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { - const cardinalityFieldsCache = initCardinalityFieldsCache(); + const fieldsService = fieldsServiceProvider(callAsCurrentUser); return async ( analysisConfig: AnalysisConfig, @@ -137,8 +59,6 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { 'mlcategory' ); - const fieldsService = fieldsServiceProvider(callAsCurrentUser); - const { detectors, influencers, bucket_span: bucketSpan } = analysisConfig; let overallCardinality = {}; @@ -171,55 +91,27 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { !overallCardinalityFields.has(influencerField) ) as string[]; - // Check if some of the values are already cached - const cachedValues = cardinalityFieldsCache.getValues( - indexPattern, - timeFieldName, - earliestMs, - latestMs, - [...overallCardinalityFields], - maxBucketFieldCardinalities - ); - overallCardinality = cachedValues?.overallCardinality ?? {}; - maxBucketCardinality = cachedValues?.maxBucketCardinality ?? {}; - - const overallCardinalityFieldsToFetch = [...overallCardinalityFields].filter( - v => !overallCardinality.hasOwnProperty(v) - ); - if (overallCardinalityFieldsToFetch.length > 0) { - const overallCardinalitResp = await fieldsService.getCardinalityOfFields( + if (overallCardinalityFields.size > 0) { + overallCardinality = await fieldsService.getCardinalityOfFields( indexPattern, - overallCardinalityFieldsToFetch, + [...overallCardinalityFields], query, timeFieldName, earliestMs, latestMs ); - overallCardinality = { ...overallCardinality, ...overallCardinalitResp }; - - cardinalityFieldsCache.updateValues(indexPattern, timeFieldName, earliestMs, latestMs, { - overallCardinality: overallCardinalitResp, - }); } - const maxBucketCardinalityFeildToFetch = maxBucketFieldCardinalities.filter( - v => !maxBucketCardinality.hasOwnProperty(v) - ); - if (maxBucketCardinalityFeildToFetch.length > 0) { - const maxBucketCardinalityResp = await fieldsService.getMaxBucketCardinalities( + if (maxBucketFieldCardinalities.length > 0) { + maxBucketCardinality = await fieldsService.getMaxBucketCardinalities( indexPattern, - maxBucketCardinalityFeildToFetch, + maxBucketFieldCardinalities, query, timeFieldName, earliestMs, latestMs, bucketSpan ); - maxBucketCardinality = { ...maxBucketCardinality, ...maxBucketCardinalityResp }; - - cardinalityFieldsCache.updateValues(indexPattern, timeFieldName, earliestMs, latestMs, { - maxBucketCardinality: maxBucketCardinalityResp, - }); } return { diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts b/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts new file mode 100644 index 0000000000000..f0efd15275d38 --- /dev/null +++ b/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { pick } from 'lodash'; + +/** + * Aggregations types that cached. + */ +type AggType = 'overallCardinality' | 'maxBucketCardinality'; + +type CacheStorage = { [key in AggType]: { [field: string]: number } }; + +/** + * Caches cardinality fields values to avoid + * unnecessary aggregations on elasticsearch + */ +export const initCardinalityFieldsCache = () => { + const cardinalityCache = new Map(); + + return { + /** + * Gets requested values from cache + */ + getValues( + indexPattern: string | string[], + timeField: string, + earliestMs: number, + latestMs: number, + aggType: AggType, + fieldNames: string[] + ): CacheStorage[AggType] | null { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cached = cardinalityCache.get(cacheKey); + if (!cached) { + return null; + } + return pick(cached[aggType], fieldNames); + }, + /** + * Extends cache with provided values + */ + updateValues( + indexPattern: string | string[], + timeField: string, + earliestMs: number, + latestMs: number, + update: Partial + ): void { + const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cachedValues = cardinalityCache.get(cacheKey); + if (cachedValues === undefined) { + cardinalityCache.set(cacheKey, { + overallCardinality: update.overallCardinality ?? {}, + maxBucketCardinality: update.maxBucketCardinality ?? {}, + }); + return; + } + + Object.assign(cachedValues.overallCardinality, update.overallCardinality); + Object.assign(cachedValues.maxBucketCardinality, update.maxBucketCardinality); + }, + }; +}; diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts index aae79706c5588..567c5d2afb7de 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_service.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_service.ts @@ -7,12 +7,15 @@ import Boom from 'boom'; import { APICaller } from 'kibana/server'; import { parseInterval } from '../../../common/util/parse_interval'; +import { initCardinalityFieldsCache } from './fields_aggs_cache'; /** * Service for carrying out queries to obtain data * specific to fields in Elasticsearch indices. */ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { + const fieldsAggsCache = initCardinalityFieldsCache(); + /** * Gets aggregatable fields. */ @@ -58,6 +61,23 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { return {}; } + const cachedValues = + fieldsAggsCache.getValues( + index, + timeFieldName, + earliestMs, + latestMs, + 'overallCardinality', + fieldNames + ) ?? {}; + + // No need to perform aggregation over the cached fields + const fieldsToAgg = aggregatableFields.filter(field => !cachedValues.hasOwnProperty(field)); + + if (fieldsToAgg.length === 0) { + return cachedValues; + } + // Build the criteria to use in the bool filter part of the request. // Add criteria for the time range and the datafeed config query. const mustCriteria = [ @@ -76,7 +96,7 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { mustCriteria.push(query); } - const aggs = aggregatableFields.reduce((obj, field) => { + const aggs = fieldsToAgg.reduce((obj, field) => { obj[field] = { cardinality: { field } }; return obj; }, {} as { [field: string]: { cardinality: { field: string } } }); @@ -105,10 +125,19 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { return {}; } - return aggregatableFields.reduce((obj, field) => { + const aggResult = fieldsToAgg.reduce((obj, field) => { obj[field] = (aggregations[field] || { value: 0 }).value; return obj; }, {} as { [field: string]: number }); + + fieldsAggsCache.updateValues(index, timeFieldName, earliestMs, latestMs, { + overallCardinality: aggResult, + }); + + return { + ...cachedValues, + ...aggResult, + }; } /** @@ -214,6 +243,23 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { return {}; } + const cachedValues = + fieldsAggsCache.getValues( + index, + timeFieldName, + earliestMs, + latestMs, + 'maxBucketCardinality', + fieldNames + ) ?? {}; + + // No need to perform aggregation over the cached fields + const fieldsToAgg = aggregatableFields.filter(field => !cachedValues.hasOwnProperty(field)); + + if (fieldsToAgg.length === 0) { + return cachedValues; + } + const { start, end } = getSafeTimeRange(earliestMs, latestMs, interval); const mustCriteria = [ @@ -239,7 +285,7 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { const getSafeAggName = (field: string) => field.replace(/\W/g, ''); const getMaxBucketAggKey = (field: string) => `max_bucket_${field}`; - const fieldsCardinalityAggs = aggregatableFields.reduce((obj, field) => { + const fieldsCardinalityAggs = fieldsToAgg.reduce((obj, field) => { obj[getSafeAggName(field)] = { cardinality: { field } }; return obj; }, {} as { [field: string]: { cardinality: { field: string } } }); @@ -280,13 +326,18 @@ export function fieldsServiceProvider(callAsCurrentUser: APICaller) { )?.aggregations; if (!aggregations) { - return {}; + return cachedValues; } - return aggregatableFields.reduce((obj, field) => { + const aggResult = fieldsToAgg.reduce((obj, field) => { obj[field] = (aggregations[getMaxBucketAggKey(field)] || { value: 0 }).value ?? 0; return obj; }, {} as { [field: string]: number }); + + return { + ...cachedValues, + ...aggResult, + }; } return { From 57cbe6958491a4fb44091639dd29f7c6db87880a Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 11:33:35 +0100 Subject: [PATCH 07/15] [ML] fix types, add comments --- .../models/data_recognizer/data_recognizer.ts | 27 ++++++++++--------- .../fields_service/fields_aggs_cache.ts | 10 +++---- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 9c4dfa81af783..de646877ad3d5 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -370,16 +370,16 @@ export class DataRecognizer { // if any of the savedObjects already exist, they will not be overwritten. async setupModuleItems( moduleId: string, - jobPrefix: string | undefined, - groups: string[] | undefined, - indexPatternName: string | undefined, - query: any, - useDedicatedIndex: boolean | undefined, - startDatafeed: boolean | undefined, - start: number | undefined, - end: number | undefined, - jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[], + jobPrefix?: string, + groups?: string[], + indexPatternName?: string, + query?: any, + useDedicatedIndex?: boolean, + startDatafeed?: boolean, + start?: number, + end?: number, + jobOverrides?: JobOverride[], + datafeedOverrides?: DatafeedOverride[], estimateModelMemory?: boolean ) { // load the config from disk @@ -982,6 +982,7 @@ export class DataRecognizer { const calculateModelMemoryLimit = calculateModelMemoryLimitProvider(this.callAsCurrentUser); const query = moduleConfig.query ?? null; + // Checks if all jobs in the module have the same time field configured const isSameTimeFields = moduleConfig.jobs.every( job => job.config.data_description.time_field === @@ -989,6 +990,8 @@ export class DataRecognizer { ); if (isSameTimeFields && (start === undefined || end === undefined)) { + // In case of time range is not provided and the time field is the same + // set the fallback range for all jobs const { start: fallbackStart, end: fallbackEnd } = await this.getFallbackTimeRange( moduleConfig.jobs[0].config.data_description.time_field, query @@ -1063,7 +1066,7 @@ export class DataRecognizer { return false; } - applyJobConfigOverrides(moduleConfig: Module, jobOverrides: JobOverride[], jobPrefix = '') { + applyJobConfigOverrides(moduleConfig: Module, jobOverrides?: JobOverride[], jobPrefix = '') { if (jobOverrides === undefined || jobOverrides === null) { return; } @@ -1140,7 +1143,7 @@ export class DataRecognizer { applyDatafeedConfigOverrides( moduleConfig: Module, - datafeedOverrides: DatafeedOverride | DatafeedOverride[], + datafeedOverrides?: DatafeedOverride | DatafeedOverride[], jobPrefix = '' ) { if (datafeedOverrides !== undefined && datafeedOverrides !== null) { diff --git a/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts b/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts index f0efd15275d38..cdaefe6fdeed7 100644 --- a/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts +++ b/x-pack/plugins/ml/server/models/fields_service/fields_aggs_cache.ts @@ -7,7 +7,7 @@ import { pick } from 'lodash'; /** - * Aggregations types that cached. + * Cached aggregation types */ type AggType = 'overallCardinality' | 'maxBucketCardinality'; @@ -25,14 +25,14 @@ export const initCardinalityFieldsCache = () => { * Gets requested values from cache */ getValues( - indexPattern: string | string[], + indexPatternName: string | string[], timeField: string, earliestMs: number, latestMs: number, aggType: AggType, fieldNames: string[] ): CacheStorage[AggType] | null { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cacheKey = indexPatternName + timeField + earliestMs + latestMs; const cached = cardinalityCache.get(cacheKey); if (!cached) { return null; @@ -43,13 +43,13 @@ export const initCardinalityFieldsCache = () => { * Extends cache with provided values */ updateValues( - indexPattern: string | string[], + indexPatternName: string | string[], timeField: string, earliestMs: number, latestMs: number, update: Partial ): void { - const cacheKey = indexPattern + timeField + earliestMs + latestMs; + const cacheKey = indexPatternName + timeField + earliestMs + latestMs; const cachedValues = cardinalityCache.get(cacheKey); if (cachedValues === undefined) { cardinalityCache.set(cacheKey, { From e59cc847030f28ef1b54685f434696100afe9468 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 13:10:30 +0100 Subject: [PATCH 08/15] [ML] check for MML overrides --- x-pack/plugins/ml/common/types/modules.ts | 8 ++-- .../models/data_recognizer/data_recognizer.ts | 37 +++++++++++++------ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index e61ff9972d601..900bde0c222c4 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -90,8 +90,10 @@ export interface DataRecognizerConfigResponse { }; } -export type GeneralOverride = any; +export type GeneralJobsOverride = Partial>; -export type JobOverride = Partial; +export type JobOverride = Partial & { job_id: Job['job_id'] }; -export type DatafeedOverride = Partial; +export type GeneralDatafeedsOverride = Partial>; + +export type DatafeedOverride = Partial & { job_id: Job['job_id'] }; diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index de646877ad3d5..5c7178fc3b48f 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -19,11 +19,12 @@ import { Module, JobOverride, DatafeedOverride, - GeneralOverride, + GeneralJobsOverride, DatafeedResponse, JobResponse, KibanaObjectResponse, DataRecognizerConfigResponse, + GeneralDatafeedsOverride, } from '../../../common/types/modules'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; @@ -111,6 +112,10 @@ export class DataRecognizer { indexPatternName: string = ''; indexPatternId: string | undefined = undefined; savedObjectsClient: SavedObjectsClientContract; + /** + * List of job ids that require model memory estimation + */ + jobsForModelMemoryEstimation: string[] = []; callAsCurrentUser: ( endpoint: string, @@ -978,29 +983,31 @@ export class DataRecognizer { return; } - if (estimateMML) { + if (estimateMML && this.jobsForModelMemoryEstimation.length > 0) { const calculateModelMemoryLimit = calculateModelMemoryLimitProvider(this.callAsCurrentUser); const query = moduleConfig.query ?? null; + const jobs = moduleConfig.jobs.filter(job => + this.jobsForModelMemoryEstimation.includes(job.id) + ); + // Checks if all jobs in the module have the same time field configured - const isSameTimeFields = moduleConfig.jobs.every( - job => - job.config.data_description.time_field === - moduleConfig.jobs[0].config.data_description.time_field + const isSameTimeFields = jobs.every( + job => job.config.data_description.time_field === jobs[0].config.data_description.time_field ); if (isSameTimeFields && (start === undefined || end === undefined)) { // In case of time range is not provided and the time field is the same // set the fallback range for all jobs const { start: fallbackStart, end: fallbackEnd } = await this.getFallbackTimeRange( - moduleConfig.jobs[0].config.data_description.time_field, + jobs[0].config.data_description.time_field, query ); start = fallbackStart; end = fallbackEnd; } - for (const job of moduleConfig.jobs) { + for (const job of jobs) { let earliestMs = start; let latestMs = end; if (earliestMs === undefined || latestMs === undefined) { @@ -1027,8 +1034,6 @@ export class DataRecognizer { job.config.analysis_limits.model_memory_limit = modelMemoryLimit; } - - return; } for (const job of moduleConfig.jobs) { @@ -1084,7 +1089,7 @@ export class DataRecognizer { // separate all the overrides. // the overrides which don't contain a job id will be applied to all jobs in the module - const generalOverrides: GeneralOverride[] = []; + const generalOverrides: GeneralJobsOverride[] = []; const jobSpecificOverrides: JobOverride[] = []; overrides.forEach(override => { @@ -1095,6 +1100,14 @@ export class DataRecognizer { } }); + if (generalOverrides.some(override => !!override.analysis_limits?.model_memory_limit)) { + this.jobsForModelMemoryEstimation = []; + } else { + this.jobsForModelMemoryEstimation = jobSpecificOverrides + .filter(override => !override.analysis_limits?.model_memory_limit) + .map(override => override.job_id); + } + function processArrayValues(source: any, update: any) { if (typeof source !== 'object' || typeof update !== 'object') { return; @@ -1160,7 +1173,7 @@ export class DataRecognizer { // separate all the overrides. // the overrides which don't contain a datafeed id or a job id will be applied to all jobs in the module - const generalOverrides: GeneralOverride[] = []; + const generalOverrides: GeneralDatafeedsOverride[] = []; const datafeedSpecificOverrides: DatafeedOverride[] = []; overrides.forEach(o => { if (o.datafeed_id === undefined && o.job_id === undefined) { From 0e3c9e1f384208ffea6ff9a18b3c52faf3877ed7 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 13:11:38 +0100 Subject: [PATCH 09/15] [ML] disable estimateModelMemory --- .../ml/public/application/jobs/new_job/recognize/page.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx index c5192f99ce48a..9b76b9be9bf45 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/recognize/page.tsx @@ -172,7 +172,7 @@ export const Page: FC = ({ moduleId, existingGroupIds }) => { startDatafeed: startDatafeedAfterSave, ...(jobOverridesPayload !== null ? { jobOverrides: jobOverridesPayload } : {}), ...resultTimeRange, - estimateModelMemory: true, + estimateModelMemory: false, }); const { datafeeds: datafeedsResponse, jobs: jobsResponse, kibana: kibanaResponse } = response; From cd8c14074b2fa29c5f6b2ccb3a7c47b5423b4e0e Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 13:59:47 +0100 Subject: [PATCH 10/15] [ML] fix typing --- x-pack/plugins/ml/common/types/modules.ts | 8 ++++++-- .../ml/server/models/data_recognizer/data_recognizer.ts | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index 900bde0c222c4..3288f06b578b1 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -90,9 +90,13 @@ export interface DataRecognizerConfigResponse { }; } -export type GeneralJobsOverride = Partial>; +export type JobOverride = Partial; +export type GeneralJobsOverride = Omit; +export type JobSpecificOverride = JobOverride & { job_id: Job['job_id'] }; -export type JobOverride = Partial & { job_id: Job['job_id'] }; +export function isGeneralJobOverride(override: JobOverride): override is GeneralJobsOverride { + return override.job_id === undefined; +} export type GeneralDatafeedsOverride = Partial>; diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 5c7178fc3b48f..2f9ccb5c45df0 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -25,6 +25,8 @@ import { KibanaObjectResponse, DataRecognizerConfigResponse, GeneralDatafeedsOverride, + JobSpecificOverride, + isGeneralJobOverride, } from '../../../common/types/modules'; import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils'; import { mlLog } from '../../client/log'; @@ -1090,10 +1092,10 @@ export class DataRecognizer { // separate all the overrides. // the overrides which don't contain a job id will be applied to all jobs in the module const generalOverrides: GeneralJobsOverride[] = []; - const jobSpecificOverrides: JobOverride[] = []; + const jobSpecificOverrides: JobSpecificOverride[] = []; overrides.forEach(override => { - if (override.job_id === undefined) { + if (isGeneralJobOverride(override)) { generalOverrides.push(override); } else { jobSpecificOverrides.push(override); From 80eaeef10586db07fa3e4580498badbb8d2f0df7 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 14:38:44 +0100 Subject: [PATCH 11/15] [ML] check for empty max mml --- .../models/data_recognizer/data_recognizer.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 2f9ccb5c45df0..3e65d4ec0d301 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -975,12 +975,6 @@ export class DataRecognizer { start?: number, end?: number ) { - const { limits } = await this.callAsCurrentUser('ml.info'); - - const maxMml = limits.max_model_memory_limit; - // @ts-ignore - const maxBytes: number = numeral(maxMml.toUpperCase()).value(); - if (!Array.isArray(moduleConfig.jobs)) { return; } @@ -1038,6 +1032,16 @@ export class DataRecognizer { } } + const { limits } = await this.callAsCurrentUser('ml.info'); + const maxMml = limits.max_model_memory_limit; + + if (!maxMml) { + return; + } + + // @ts-ignore + const maxBytes: number = numeral(maxMml.toUpperCase()).value(); + for (const job of moduleConfig.jobs) { const mml = job.config?.analysis_limits?.model_memory_limit; if (mml !== undefined) { From 18f537cfd51ee6990ec270f44400d6da1cd87ec3 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 15:27:04 +0100 Subject: [PATCH 12/15] [ML] refactor, update types, fix jobsForModelMemoryEstimation --- x-pack/plugins/ml/common/types/modules.ts | 2 +- .../models/data_recognizer/data_recognizer.ts | 53 +++++++++---------- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index 3288f06b578b1..55c5a05204e00 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -98,6 +98,6 @@ export function isGeneralJobOverride(override: JobOverride): override is General return override.job_id === undefined; } -export type GeneralDatafeedsOverride = Partial>; +export type GeneralDatafeedsOverride = Partial>; export type DatafeedOverride = Partial & { job_id: Job['job_id'] }; diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 3e65d4ec0d301..807d03354b1a6 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -7,11 +7,12 @@ import fs from 'fs'; import Boom from 'boom'; import numeral from '@elastic/numeral'; -import { CallAPIOptions, APICaller, SavedObjectsClientContract } from 'kibana/server'; +import { APICaller, SavedObjectsClientContract } from 'kibana/server'; import moment from 'moment'; import { IndexPatternAttributes } from 'src/plugins/data/server'; import { merge } from 'lodash'; import { AnalysisLimits, CombinedJobWithStats } from '../../../common/types/anomaly_detection_jobs'; +import { MlInfoResponse } from '../../../common/types/ml_server_info'; import { KibanaObjects, ModuleDataFeed, @@ -113,22 +114,15 @@ export class DataRecognizer { modulesDir = `${__dirname}/modules`; indexPatternName: string = ''; indexPatternId: string | undefined = undefined; - savedObjectsClient: SavedObjectsClientContract; /** * List of job ids that require model memory estimation */ - jobsForModelMemoryEstimation: string[] = []; + jobsForModelMemoryEstimation: ModuleJob[] = []; - callAsCurrentUser: ( - endpoint: string, - clientParams?: Record, - options?: CallAPIOptions - ) => Promise; - - constructor(callAsCurrentUser: APICaller, savedObjectsClient: SavedObjectsClientContract) { - this.callAsCurrentUser = callAsCurrentUser; - this.savedObjectsClient = savedObjectsClient; - } + constructor( + private callAsCurrentUser: APICaller, + private savedObjectsClient: SavedObjectsClientContract + ) {} // list all directories under the given directory async listDirs(dirName: string): Promise { @@ -385,8 +379,8 @@ export class DataRecognizer { startDatafeed?: boolean, start?: number, end?: number, - jobOverrides?: JobOverride[], - datafeedOverrides?: DatafeedOverride[], + jobOverrides?: JobOverride | JobOverride[], + datafeedOverrides?: DatafeedOverride | DatafeedOverride[], estimateModelMemory?: boolean ) { // load the config from disk @@ -983,27 +977,25 @@ export class DataRecognizer { const calculateModelMemoryLimit = calculateModelMemoryLimitProvider(this.callAsCurrentUser); const query = moduleConfig.query ?? null; - const jobs = moduleConfig.jobs.filter(job => - this.jobsForModelMemoryEstimation.includes(job.id) - ); - // Checks if all jobs in the module have the same time field configured - const isSameTimeFields = jobs.every( - job => job.config.data_description.time_field === jobs[0].config.data_description.time_field + const isSameTimeFields = this.jobsForModelMemoryEstimation.every( + job => + job.config.data_description.time_field === + this.jobsForModelMemoryEstimation[0].config.data_description.time_field ); if (isSameTimeFields && (start === undefined || end === undefined)) { // In case of time range is not provided and the time field is the same // set the fallback range for all jobs const { start: fallbackStart, end: fallbackEnd } = await this.getFallbackTimeRange( - jobs[0].config.data_description.time_field, + this.jobsForModelMemoryEstimation[0].config.data_description.time_field, query ); start = fallbackStart; end = fallbackEnd; } - for (const job of jobs) { + for (const job of this.jobsForModelMemoryEstimation) { let earliestMs = start; let latestMs = end; if (earliestMs === undefined || latestMs === undefined) { @@ -1032,7 +1024,7 @@ export class DataRecognizer { } } - const { limits } = await this.callAsCurrentUser('ml.info'); + const { limits } = await this.callAsCurrentUser('ml.info'); const maxMml = limits.max_model_memory_limit; if (!maxMml) { @@ -1077,7 +1069,11 @@ export class DataRecognizer { return false; } - applyJobConfigOverrides(moduleConfig: Module, jobOverrides?: JobOverride[], jobPrefix = '') { + applyJobConfigOverrides( + moduleConfig: Module, + jobOverrides?: JobOverride | JobOverride[], + jobPrefix = '' + ) { if (jobOverrides === undefined || jobOverrides === null) { return; } @@ -1109,9 +1105,10 @@ export class DataRecognizer { if (generalOverrides.some(override => !!override.analysis_limits?.model_memory_limit)) { this.jobsForModelMemoryEstimation = []; } else { - this.jobsForModelMemoryEstimation = jobSpecificOverrides - .filter(override => !override.analysis_limits?.model_memory_limit) - .map(override => override.job_id); + this.jobsForModelMemoryEstimation = moduleConfig.jobs.filter(job => { + const override = jobSpecificOverrides.find(o => o.job_id === job.id); + return override?.analysis_limits?.model_memory_limit === undefined; + }); } function processArrayValues(source: any, update: any) { From c8fd7fcde62c7c6273c20621a53adf4430c2b17e Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 15:33:59 +0100 Subject: [PATCH 13/15] [ML] fix override lookup --- .../plugins/ml/server/models/data_recognizer/data_recognizer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 807d03354b1a6..871d1ea50c087 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -1106,7 +1106,7 @@ export class DataRecognizer { this.jobsForModelMemoryEstimation = []; } else { this.jobsForModelMemoryEstimation = moduleConfig.jobs.filter(job => { - const override = jobSpecificOverrides.find(o => o.job_id === job.id); + const override = jobSpecificOverrides.find(o => `${jobPrefix}${o.job_id}` === job.id); return override?.analysis_limits?.model_memory_limit === undefined; }); } From 085960a936a2882f011597b75bc88831d504de73 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 17:37:29 +0100 Subject: [PATCH 14/15] [ML] resolve nit comments --- x-pack/plugins/ml/common/types/modules.ts | 2 +- .../calculate_model_memory_limit.ts | 3 ++- .../models/data_recognizer/data_recognizer.ts | 2 +- x-pack/plugins/ml/server/routes/modules.ts | 22 +++++++++---------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ml/common/types/modules.ts b/x-pack/plugins/ml/common/types/modules.ts index 55c5a05204e00..b476762f6efca 100644 --- a/x-pack/plugins/ml/common/types/modules.ts +++ b/x-pack/plugins/ml/common/types/modules.ts @@ -100,4 +100,4 @@ export function isGeneralJobOverride(override: JobOverride): override is General export type GeneralDatafeedsOverride = Partial>; -export type DatafeedOverride = Partial & { job_id: Job['job_id'] }; +export type DatafeedOverride = Partial; diff --git a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts index 70bbc110ac050..cd61dd9eddcdd 100644 --- a/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts +++ b/x-pack/plugins/ml/server/models/calculate_model_memory_limit/calculate_model_memory_limit.ts @@ -6,6 +6,7 @@ import numeral from '@elastic/numeral'; import { APICaller } from 'kibana/server'; +import { MLCATEGORY } from '../../../common/constants/field_types'; import { AnalysisConfig } from '../../../common/types/anomaly_detection_jobs'; import { fieldsServiceProvider } from '../fields_service'; @@ -56,7 +57,7 @@ const cardinalityCheckProvider = (callAsCurrentUser: APICaller) => { * The keyword which is used to mean the output of categorization, * so it will have cardinality zero in the actual input data. */ - 'mlcategory' + MLCATEGORY ); const { detectors, influencers, bucket_span: bucketSpan } = analysisConfig; diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index 871d1ea50c087..b86c6e7d4546e 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -115,7 +115,7 @@ export class DataRecognizer { indexPatternName: string = ''; indexPatternId: string | undefined = undefined; /** - * List of job ids that require model memory estimation + * List of the module jobs that require model memory estimation */ jobsForModelMemoryEstimation: ModuleJob[] = []; diff --git a/x-pack/plugins/ml/server/routes/modules.ts b/x-pack/plugins/ml/server/routes/modules.ts index 797f943031625..358cd0ac2871c 100644 --- a/x-pack/plugins/ml/server/routes/modules.ts +++ b/x-pack/plugins/ml/server/routes/modules.ts @@ -36,17 +36,17 @@ function getModule(context: RequestHandlerContext, moduleId: string) { function saveModuleItems( context: RequestHandlerContext, moduleId: string, - prefix: string | undefined, - groups: string[] | undefined, - indexPatternName: string | undefined, - query: any, - useDedicatedIndex: boolean | undefined, - startDatafeed: boolean | undefined, - start: number | undefined, - end: number | undefined, - jobOverrides: JobOverride[], - datafeedOverrides: DatafeedOverride[], - estimateModelMemory: boolean | undefined + prefix?: string, + groups?: string[], + indexPatternName?: string, + query?: any, + useDedicatedIndex?: boolean, + startDatafeed?: boolean, + start?: number, + end?: number, + jobOverrides?: JobOverride | JobOverride[], + datafeedOverrides?: DatafeedOverride | DatafeedOverride[], + estimateModelMemory?: boolean ) { const dr = new DataRecognizer( context.ml!.mlClient.callAsCurrentUser, From dde330ba6fe4813e9edc84f03966c5acb7ffd737 Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Mon, 23 Mar 2020 18:44:19 +0100 Subject: [PATCH 15/15] [ML] init jobsForModelMemoryEstimation --- .../plugins/ml/server/models/data_recognizer/data_recognizer.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts index b86c6e7d4546e..824f9cc57982c 100644 --- a/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts +++ b/x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts @@ -423,6 +423,8 @@ export class DataRecognizer { savedObjects: [] as KibanaObjectResponse[], }; + this.jobsForModelMemoryEstimation = moduleConfig.jobs; + this.applyJobConfigOverrides(moduleConfig, jobOverrides, jobPrefix); this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix); this.updateDatafeedIndices(moduleConfig);