From 9714423662978fe259ff6e65771dc99f415297e1 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Fri, 6 Oct 2023 16:02:20 +0200 Subject: [PATCH 01/11] [infra] Shorten IDs for ML jobs (#47477) --- .../results/v1/log_entry_anomalies.ts | 2 + .../v1/log_entry_anomalies_datasets.ts | 2 + .../results/v1/log_entry_categories.ts | 2 + .../results/v1/log_entry_category_datasets.ts | 2 + .../results/v1/log_entry_category_examples.ts | 2 + .../results/v1/log_entry_examples.ts | 2 + .../http_api/log_analysis/v1/id_formats.ts | 39 +++++++++ .../common/log_analysis/job_parameters.ts | 33 +++++-- .../log_entry_categories_analysis.ts | 6 +- .../log_analysis/log_entry_rate_analysis.ts | 7 +- .../setup_flyout/module_list.tsx | 5 +- .../logs/log_analysis/api/ml_cleanup.ts | 29 ++++--- .../api/ml_get_jobs_summary_api.ts | 14 +-- .../log_analysis/api/ml_setup_module_api.ts | 2 +- .../log_analysis/log_analysis_cleanup.tsx | 29 ++++--- .../logs/log_analysis/log_analysis_module.tsx | 36 ++++++-- .../log_analysis_module_configuration.ts | 9 +- .../log_analysis_module_definition.tsx | 11 ++- .../log_analysis_module_status.tsx | 55 +++++++----- .../log_analysis/log_analysis_module_types.ts | 9 +- .../log_analysis/log_analysis_setup_state.ts | 9 +- .../log_entry_categories/module_descriptor.ts | 36 ++++++-- .../use_log_entry_categories_module.tsx | 5 ++ .../log_entry_rate/module_descriptor.ts | 30 +++++-- .../use_log_entry_rate_module.tsx | 5 ++ .../public/containers/ml/api/ml_get_module.ts | 2 +- .../ml/infra_ml_module_definition.tsx | 2 +- .../pages/logs/log_entry_categories/page.tsx | 9 +- .../log_entry_categories/page_content.tsx | 8 +- .../log_entry_categories/page_providers.tsx | 9 +- .../page_results_content.tsx | 10 ++- .../top_categories/category_details_row.tsx | 5 ++ .../get_log_entry_category_datasets.ts | 5 +- .../get_log_entry_category_examples.ts | 5 +- .../get_top_log_entry_categories.ts | 6 +- .../use_log_entry_categories_results.ts | 18 +++- .../use_log_entry_category_examples.tsx | 6 +- .../public/pages/logs/log_entry_rate/page.tsx | 9 +- .../logs/log_entry_rate/page_content.tsx | 15 +++- .../logs/log_entry_rate/page_providers.tsx | 11 ++- .../log_entry_rate/page_results_content.tsx | 19 +++-- .../sections/anomalies/expanded_row.tsx | 9 +- .../service_calls/get_log_entry_anomalies.ts | 6 +- .../get_log_entry_anomalies_datasets.ts | 5 +- .../service_calls/get_log_entry_examples.ts | 6 +- .../use_log_entry_anomalies_results.ts | 9 +- .../log_entry_rate/use_log_entry_examples.ts | 6 +- .../call_get_log_analysis_id_formats.ts | 37 ++++++++ .../shared/use_log_ml_job_id_formats_shim.tsx | 85 +++++++++++++++++++ x-pack/plugins/infra/server/infra_server.ts | 2 + .../lib/log_analysis/log_entry_anomalies.ts | 30 ++++++- .../log_entry_categories_analysis.ts | 7 ++ .../log_analysis/log_entry_rate_analysis.ts | 6 +- .../lib/log_analysis/resolve_id_formats.ts | 77 +++++++++++++++++ .../server/routes/log_analysis/id_formats.ts | 73 ++++++++++++++++ .../infra/server/routes/log_analysis/index.ts | 1 + .../results/log_entry_anomalies.ts | 2 + .../results/log_entry_anomalies_datasets.ts | 2 + .../results/log_entry_categories.ts | 2 + .../results/log_entry_category_datasets.ts | 2 + .../results/log_entry_category_examples.ts | 2 + .../results/log_entry_examples.ts | 2 + 62 files changed, 735 insertions(+), 156 deletions(-) create mode 100644 x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts create mode 100644 x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts create mode 100644 x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx create mode 100644 x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts create mode 100644 x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts index 3553962063990..d5463185991c4 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts @@ -8,6 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; +import { idFormatByJobTypeRT } from '../../v1/id_formats'; import { timeRangeRT, routeTimingMetadataRT } from '../../../shared'; import { logEntryAnomalyRT, @@ -54,6 +55,7 @@ export const getLogEntryAnomaliesRequestPayloadRT = rt.type({ rt.type({ // log view logView: persistedLogViewReferenceRT, + idFormats: idFormatByJobTypeRT, // the time range to fetch the log entry anomalies from timeRange: timeRangeRT, }), diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts index c07007be05115..22460385d7c02 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts @@ -14,6 +14,7 @@ import { timeRangeRT, routeTimingMetadataRT, } from '../../../shared'; +import { idFormatByJobTypeRT } from '../../v1/id_formats'; export const LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_DATASETS_PATH = '/api/infra/log_analysis/results/log_entry_anomalies_datasets'; @@ -26,6 +27,7 @@ export const getLogEntryAnomaliesDatasetsRequestPayloadRT = rt.type({ data: rt.type({ // log view logView: persistedLogViewReferenceRT, + idFormats: idFormatByJobTypeRT, // the time range to fetch the anomalies datasets from timeRange: timeRangeRT, }), diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts index e84825b8c6835..0e0f2f74e6e38 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts @@ -8,6 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; +import { idFormatRT } from '../../v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, @@ -41,6 +42,7 @@ export const getLogEntryCategoriesRequestPayloadRT = rt.type({ categoryCount: rt.number, // log view logView: persistedLogViewReferenceRT, + idFormat: idFormatRT, // the time range to fetch the categories from timeRange: timeRangeRT, // a list of histograms to create diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts index e051e313d9b8e..48d4b8c9f8ae7 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts @@ -8,6 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; +import { idFormatRT } from '../../v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, @@ -25,6 +26,7 @@ export const getLogEntryCategoryDatasetsRequestPayloadRT = rt.type({ data: rt.type({ // log view logView: persistedLogViewReferenceRT, + idFormat: idFormatRT, // the time range to fetch the category datasets from timeRange: timeRangeRT, }), diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts index fc6ece5d7b7f7..8eb4baef59f13 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts @@ -7,6 +7,7 @@ import { logEntryContextRT, persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; import * as rt from 'io-ts'; +import { idFormatRT } from '../../v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, @@ -29,6 +30,7 @@ export const getLogEntryCategoryExamplesRequestPayloadRT = rt.type({ exampleCount: rt.number, // log view logView: persistedLogViewReferenceRT, + idFormat: idFormatRT, // the time range to fetch the category examples from timeRange: timeRangeRT, }), diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts index ebc78693f4983..5f61d0be72ace 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts @@ -7,6 +7,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; +import { idFormatRT } from '../../v1/id_formats'; import { logEntryExampleRT } from '../../../../log_analysis'; import { badRequestErrorRT, @@ -31,6 +32,7 @@ export const getLogEntryExamplesRequestPayloadRT = rt.type({ exampleCount: rt.number, // logView logView: persistedLogViewReferenceRT, + idFormat: idFormatRT, // the time range to fetch the log rate examples from timeRange: timeRangeRT, }), diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts b/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts new file mode 100644 index 0000000000000..1e323d44d0096 --- /dev/null +++ b/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as rt from 'io-ts'; +import { logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT } from '../../../log_analysis'; + +export const idFormatRT = rt.union([rt.literal('legacy'), rt.literal('hashed')]); +export type IdFormat = rt.TypeOf; + +const jobTypeRT = rt.union([logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT]); +export type JobType = rt.TypeOf; + +export const idFormatByJobTypeRT = rt.record(jobTypeRT, idFormatRT); +export type IdFormatByJobType = rt.TypeOf; + +export const LOG_ANALYSIS_GET_ID_FORMATS = '/api/infra/log_analysis/id_formats'; + +export const getLogAnalysisIdFormatsRequestPayloadRT = rt.type({ + data: rt.type({ + logViewId: rt.string, + spaceId: rt.string, + }), +}); + +export type GetLogAnalysisIdFormatsRequestPayload = rt.TypeOf< + typeof getLogAnalysisIdFormatsRequestPayloadRT +>; + +export const getLogAnalysisIdFormatsSuccessResponsePayloadRT = rt.type({ + data: rt.record(rt.union([logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT]), idFormatRT), +}); + +export type GetLogAnalysisIdFormatsSuccessResponsePayload = rt.TypeOf< + typeof getLogAnalysisIdFormatsSuccessResponsePayloadRT +>; diff --git a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts index 1a695af80a4d5..223304d171a4e 100644 --- a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts @@ -6,6 +6,8 @@ */ import * as rt from 'io-ts'; +import { createHash } from 'crypto'; +import { IdFormat, JobType } from '../http_api/log_analysis/v1/id_formats'; export const bucketSpan = 900000; @@ -13,14 +15,31 @@ export const categoriesMessageField = 'message'; export const partitionField = 'event.dataset'; -export const getJobIdPrefix = (spaceId: string, sourceId: string) => - `kibana-logs-ui-${spaceId}-${sourceId}-`; - -export const getJobId = (spaceId: string, logViewId: string, jobType: string) => - `${getJobIdPrefix(spaceId, logViewId)}${jobType}`; +export const getJobIdPrefix = (spaceId: string, sourceId: string, idFormat: IdFormat) => { + if (idFormat === 'legacy') { + return `kibana-logs-ui-${spaceId}-${sourceId}-`; + } else { + const hash = createHash('sha256') + .update(`${spaceId}-${sourceId}`) + .digest('hex') + .substring(0, 10); + return `logs-${hash}-`; + } +}; -export const getDatafeedId = (spaceId: string, logViewId: string, jobType: string) => - `datafeed-${getJobId(spaceId, logViewId, jobType)}`; +export const getJobId = ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + jobType: JobType +) => `${getJobIdPrefix(spaceId, logViewId, idFormat)}${jobType}`; + +export const getDatafeedId = ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + jobType: JobType +) => `datafeed-${getJobId(spaceId, logViewId, idFormat, jobType)}`; export const datasetFilterRT = rt.union([ rt.strict({ diff --git a/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts index 1ea952d7a8a15..d22c75dc3cf56 100644 --- a/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts +++ b/x-pack/plugins/infra/common/log_analysis/log_entry_categories_analysis.ts @@ -8,9 +8,7 @@ import * as rt from 'io-ts'; import { sortRT } from './log_analysis_results'; -export const logEntryCategoriesJobTypeRT = rt.keyof({ - 'log-entry-categories-count': null, -}); +export const logEntryCategoriesJobTypeRT = rt.literal('log-entry-categories-count'); export type LogEntryCategoriesJobType = rt.TypeOf; @@ -18,6 +16,8 @@ export const logEntryCategoriesJobTypes: LogEntryCategoriesJobType[] = [ 'log-entry-categories-count', ]; +export const logEntryCategoriesJobType: LogEntryCategoriesJobType = 'log-entry-categories-count'; + export const logEntryCategoryDatasetRT = rt.type({ name: rt.string, maximumAnomalyScore: rt.number, diff --git a/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts b/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts index b13cf58764e3b..e634f5fc4ce86 100644 --- a/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts +++ b/x-pack/plugins/infra/common/log_analysis/log_entry_rate_analysis.ts @@ -7,10 +7,9 @@ import * as rt from 'io-ts'; -export const logEntryRateJobTypeRT = rt.keyof({ - 'log-entry-rate': null, -}); +export const logEntryRateJobTypeRT = rt.literal('log-entry-rate'); export type LogEntryRateJobType = rt.TypeOf; -export const logEntryRateJobTypes: LogEntryRateJobType[] = ['log-entry-rate']; +export const logEntryRateJobType: LogEntryRateJobType = 'log-entry-rate'; +export const logEntryRateJobTypes: LogEntryRateJobType[] = [logEntryRateJobType]; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx index 0ccd208cf24bd..0eec13f778e46 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/module_list.tsx @@ -7,6 +7,7 @@ import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React, { useCallback } from 'react'; +import { logEntryCategoriesJobType, logEntryRateJobType } from '../../../../../common/log_analysis'; import { useLogAnalysisCapabilitiesContext } from '../../../../containers/logs/log_analysis'; import { logEntryCategoriesModule, @@ -40,7 +41,7 @@ export const LogAnalysisModuleList: React.FC<{ { +interface DeleteJobsRequestArgs { spaceId: string; logViewId: string; - jobTypes: JobType[]; + idFormat: IdFormat; + jobTypes: T[]; } -export const callDeleteJobs = async ( - requestArgs: DeleteJobsRequestArgs, +export const callDeleteJobs = async ( + requestArgs: DeleteJobsRequestArgs, fetch: HttpHandler ) => { - const { spaceId, logViewId, jobTypes } = requestArgs; + const { spaceId, logViewId, idFormat, jobTypes } = requestArgs; // NOTE: Deleting the jobs via this API will delete the datafeeds at the same time const deleteJobsResponse = await fetch('/internal/ml/jobs/delete_jobs', { @@ -29,7 +31,7 @@ export const callDeleteJobs = async ( version: '1', body: JSON.stringify( deleteJobsRequestPayloadRT.encode({ - jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, jobType)), + jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, idFormat, jobType)), }) ), }); @@ -45,17 +47,18 @@ export const callGetJobDeletionTasks = async (fetch: HttpHandler) => { return decodeOrThrow(getJobDeletionTasksResponsePayloadRT)(jobDeletionTasksResponse); }; -interface StopDatafeedsRequestArgs { +interface StopDatafeedsRequestArgs { spaceId: string; logViewId: string; - jobTypes: JobType[]; + idFormat: IdFormat; + jobTypes: T[]; } -export const callStopDatafeeds = async ( - requestArgs: StopDatafeedsRequestArgs, +export const callStopDatafeeds = async ( + requestArgs: StopDatafeedsRequestArgs, fetch: HttpHandler ) => { - const { spaceId, logViewId, jobTypes } = requestArgs; + const { spaceId, logViewId, idFormat, jobTypes } = requestArgs; // Stop datafeed due to https://github.com/elastic/kibana/issues/44652 const stopDatafeedResponse = await fetch('/internal/ml/jobs/stop_datafeeds', { @@ -63,7 +66,9 @@ export const callStopDatafeeds = async ( version: '1', body: JSON.stringify( stopDatafeedsRequestPayloadRT.encode({ - datafeedIds: jobTypes.map((jobType) => getDatafeedId(spaceId, logViewId, jobType)), + datafeedIds: jobTypes.map((jobType) => + getDatafeedId(spaceId, logViewId, idFormat, jobType) + ), }) ), }); diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts index 35c678f7b20c0..a684eba1ac9b7 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts @@ -8,26 +8,28 @@ import * as rt from 'io-ts'; import type { HttpHandler } from '@kbn/core/public'; +import { IdFormat, JobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getJobId, jobCustomSettingsRT } from '../../../../../common/log_analysis'; import { decodeOrThrow } from '../../../../../common/runtime_types'; -interface RequestArgs { +interface RequestArgs { spaceId: string; logViewId: string; - jobTypes: JobType[]; + idFormat: IdFormat; + jobTypes: T[]; } -export const callJobsSummaryAPI = async ( - requestArgs: RequestArgs, +export const callJobsSummaryAPI = async ( + requestArgs: RequestArgs, fetch: HttpHandler ) => { - const { spaceId, logViewId, jobTypes } = requestArgs; + const { spaceId, logViewId, idFormat, jobTypes } = requestArgs; const response = await fetch('/internal/ml/jobs/jobs_summary', { method: 'POST', version: '1', body: JSON.stringify( fetchJobStatusRequestPayloadRT.encode({ - jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, jobType)), + jobIds: jobTypes.map((jobType) => getJobId(spaceId, logViewId, idFormat, jobType)), }) ), }); diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts index f19c754ada380..f1e34cdd40518 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts @@ -46,7 +46,7 @@ export const callSetupMlModuleAPI = async (requestArgs: RequestArgs, fetch: Http start, end, indexPatternName: indexPattern, - prefix: getJobIdPrefix(spaceId, sourceId), + prefix: getJobIdPrefix(spaceId, sourceId, 'hashed'), startDatafeed: true, jobOverrides, datafeedOverrides, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx index b9668311df062..b8aa4a6f810da 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx @@ -6,17 +6,19 @@ */ import type { HttpHandler } from '@kbn/core/public'; +import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { getJobId } from '../../../../common/log_analysis'; import { callDeleteJobs, callGetJobDeletionTasks, callStopDatafeeds } from './api/ml_cleanup'; -export const cleanUpJobsAndDatafeeds = async ( +export const cleanUpJobsAndDatafeeds = async ( spaceId: string, logViewId: string, - jobTypes: JobType[], + idFormat: IdFormat, + jobTypes: T[], fetch: HttpHandler ) => { try { - await callStopDatafeeds({ spaceId, logViewId, jobTypes }, fetch); + await callStopDatafeeds({ spaceId, logViewId, idFormat, jobTypes }, fetch); } catch (err) { // Proceed only if datafeed has been deleted or didn't exist in the first place if (err?.response?.status !== 404) { @@ -24,27 +26,32 @@ export const cleanUpJobsAndDatafeeds = async ( } } - return await deleteJobs(spaceId, logViewId, jobTypes, fetch); + return await deleteJobs(spaceId, logViewId, idFormat, jobTypes, fetch); }; -const deleteJobs = async ( +const deleteJobs = async ( spaceId: string, logViewId: string, - jobTypes: JobType[], + idFormat: IdFormat, + jobTypes: T[], fetch: HttpHandler ) => { - const deleteJobsResponse = await callDeleteJobs({ spaceId, logViewId, jobTypes }, fetch); - await waitUntilJobsAreDeleted(spaceId, logViewId, jobTypes, fetch); + const deleteJobsResponse = await callDeleteJobs( + { spaceId, logViewId, idFormat, jobTypes }, + fetch + ); + await waitUntilJobsAreDeleted(spaceId, logViewId, idFormat, jobTypes, fetch); return deleteJobsResponse; }; -const waitUntilJobsAreDeleted = async ( +const waitUntilJobsAreDeleted = async ( spaceId: string, logViewId: string, - jobTypes: JobType[], + idFormat: IdFormat, + jobTypes: T[], fetch: HttpHandler ) => { - const moduleJobIds = jobTypes.map((jobType) => getJobId(spaceId, logViewId, jobType)); + const moduleJobIds = jobTypes.map((jobType) => getJobId(spaceId, logViewId, idFormat, jobType)); while (true) { const { jobs } = await callGetJobDeletionTasks(fetch); const needToWait = jobs diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx index 58bade0a81d06..a0618cc20231a 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx @@ -7,22 +7,28 @@ import { useCallback, useMemo } from 'react'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; +import { useLogMlJobIdFormatsShimContext } from '../../../pages/logs/shared/use_log_ml_job_id_formats_shim'; +import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { DatasetFilter } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; import { useModuleStatus } from './log_analysis_module_status'; import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types'; -export const useLogAnalysisModule = ({ +export const useLogAnalysisModule = ({ sourceConfiguration, + idFormat, moduleDescriptor, }: { sourceConfiguration: ModuleSourceConfiguration; - moduleDescriptor: ModuleDescriptor; + idFormat: IdFormat; + moduleDescriptor: ModuleDescriptor; }) => { const { services } = useKibanaContextForPlugin(); const { spaceId, sourceId: logViewId, timestampField, runtimeMappings } = sourceConfiguration; const [moduleStatus, dispatchModuleStatus] = useModuleStatus(moduleDescriptor.jobTypes); + const { migrateIdFormat } = useLogMlJobIdFormatsShimContext(); const trackMetric = useUiTracker({ app: 'infra_logs' }); @@ -31,7 +37,12 @@ export const useLogAnalysisModule = ({ cancelPreviousOn: 'resolution', createPromise: async () => { dispatchModuleStatus({ type: 'fetchingJobStatuses' }); - return await moduleDescriptor.getJobSummary(spaceId, logViewId, services.http.fetch); + return await moduleDescriptor.getJobSummary( + spaceId, + logViewId, + idFormat, + services.http.fetch + ); }, onResolve: (jobResponse) => { dispatchModuleStatus({ @@ -39,13 +50,14 @@ export const useLogAnalysisModule = ({ payload: jobResponse, spaceId, logViewId, + idFormat, }); }, onReject: () => { dispatchModuleStatus({ type: 'failedFetchingJobStatuses' }); }, }, - [spaceId, logViewId] + [spaceId, logViewId, idFormat] ); const [, setUpModule] = useTrackedPromise( @@ -74,6 +86,7 @@ export const useLogAnalysisModule = ({ const jobSummaries = await moduleDescriptor.getJobSummary( spaceId, logViewId, + 'hashed', services.http.fetch ); return { setupResult, jobSummaries }; @@ -105,7 +118,9 @@ export const useLogAnalysisModule = ({ jobSummaries, spaceId, logViewId, + idFormat: 'hashed', }); + migrateIdFormat(moduleDescriptor.jobTypes[0]); }, onReject: (e: any) => { dispatchModuleStatus({ type: 'failedSetup' }); @@ -121,13 +136,18 @@ export const useLogAnalysisModule = ({ { cancelPreviousOn: 'resolution', createPromise: async () => { - return await moduleDescriptor.cleanUpModule(spaceId, logViewId, services.http.fetch); + return await moduleDescriptor.cleanUpModule( + spaceId, + logViewId, + idFormat, + services.http.fetch + ); }, onReject: (e) => { throw new Error(`Failed to clean up previous ML job: ${e}`); }, }, - [spaceId, logViewId] + [spaceId, logViewId, idFormat] ); const isCleaningUp = useMemo( @@ -159,8 +179,8 @@ export const useLogAnalysisModule = ({ }, [dispatchModuleStatus]); const jobIds = useMemo( - () => moduleDescriptor.getJobIds(spaceId, logViewId), - [moduleDescriptor, spaceId, logViewId] + () => moduleDescriptor.getJobIds(spaceId, logViewId, idFormat), + [moduleDescriptor, spaceId, logViewId, idFormat] ); return { diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts index 057580679210a..6e1927e3f7ea6 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts @@ -7,14 +7,15 @@ import { useMemo } from 'react'; import equal from 'fast-deep-equal'; +import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { JobSummary } from './api/ml_get_jobs_summary_api'; import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types'; -export const useLogAnalysisModuleConfiguration = ({ +export const useLogAnalysisModuleConfiguration = ({ moduleDescriptor, sourceConfiguration, }: { - moduleDescriptor: ModuleDescriptor; + moduleDescriptor: ModuleDescriptor; sourceConfiguration: ModuleSourceConfiguration; }) => { const getIsJobConfigurationOutdated = useMemo( @@ -28,8 +29,8 @@ export const useLogAnalysisModuleConfiguration = ({ }; export const isJobConfigurationOutdated = - ( - { bucketSpan }: ModuleDescriptor, + ( + { bucketSpan }: ModuleDescriptor, currentSourceConfiguration: ModuleSourceConfiguration ) => (jobSummary: JobSummary): boolean => { diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx index 61de3681b574d..818516b8cc30d 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx @@ -6,6 +6,7 @@ */ import { useCallback, useMemo, useState } from 'react'; +import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { getJobId } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; @@ -13,12 +14,14 @@ import { JobSummary } from './api/ml_get_jobs_summary_api'; import { GetMlModuleResponsePayload, JobDefinition } from './api/ml_get_module'; import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types'; -export const useLogAnalysisModuleDefinition = ({ +export const useLogAnalysisModuleDefinition = ({ sourceConfiguration: { spaceId, sourceId }, + idFormat, moduleDescriptor, }: { sourceConfiguration: ModuleSourceConfiguration; - moduleDescriptor: ModuleDescriptor; + idFormat: IdFormat; + moduleDescriptor: ModuleDescriptor; }) => { const { services } = useKibanaContextForPlugin(); const [moduleDefinition, setModuleDefinition] = useState< @@ -31,12 +34,12 @@ export const useLogAnalysisModuleDefinition = ({ ? moduleDefinition.jobs.reduce>( (accumulatedJobDefinitions, jobDefinition) => ({ ...accumulatedJobDefinitions, - [getJobId(spaceId, sourceId, jobDefinition.id)]: jobDefinition, + [getJobId(spaceId, sourceId, idFormat, jobDefinition.id as T)]: jobDefinition, }), {} ) : {}, - [moduleDefinition, sourceId, spaceId] + [moduleDefinition, sourceId, spaceId, idFormat] ); const [fetchModuleDefinitionRequest, fetchModuleDefinition] = useTrackedPromise( diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx index 6fcfea1038a04..724ec3b19e31e 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx @@ -7,6 +7,7 @@ import { useReducer } from 'react'; +import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { JobStatus, getDatafeedId, @@ -18,8 +19,8 @@ import { FetchJobStatusResponsePayload, JobSummary } from './api/ml_get_jobs_sum import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api'; import { MandatoryProperty } from '../../../../common/utility_types'; -interface StatusReducerState { - jobStatus: Record; +interface StatusReducerState { + jobStatus: Record; jobSummaries: JobSummary[]; lastSetupErrorMessages: string[]; setupStatus: SetupStatus; @@ -31,6 +32,7 @@ type StatusReducerAction = type: 'finishedSetup'; logViewId: string; spaceId: string; + idFormat: IdFormat; jobSetupResults: SetupMlModuleResponsePayload['jobs']; jobSummaries: FetchJobStatusResponsePayload; datafeedSetupResults: SetupMlModuleResponsePayload['datafeeds']; @@ -41,22 +43,23 @@ type StatusReducerAction = type: 'fetchedJobStatuses'; spaceId: string; logViewId: string; + idFormat: IdFormat; payload: FetchJobStatusResponsePayload; } | { type: 'failedFetchingJobStatuses' } | { type: 'viewedResults' }; -const createInitialState = ({ +const createInitialState = ({ jobTypes, }: { - jobTypes: JobType[]; -}): StatusReducerState => ({ + jobTypes: T[]; +}): StatusReducerState => ({ jobStatus: jobTypes.reduce( (accumulatedJobStatus, jobType) => ({ ...accumulatedJobStatus, [jobType]: 'unknown', }), - {} as Record + {} as Record ), jobSummaries: [], lastSetupErrorMessages: [], @@ -64,11 +67,8 @@ const createInitialState = ({ }); const createStatusReducer = - (jobTypes: JobType[]) => - ( - state: StatusReducerState, - action: StatusReducerAction - ): StatusReducerState => { + (jobTypes: T[]) => + (state: StatusReducerState, action: StatusReducerAction): StatusReducerState => { switch (action.type) { case 'startedSetup': { return { @@ -78,25 +78,34 @@ const createStatusReducer = ...accumulatedJobStatus, [jobType]: 'initializing', }), - {} as Record + {} as Record ), setupStatus: { type: 'pending' }, }; } case 'finishedSetup': { - const { datafeedSetupResults, jobSetupResults, jobSummaries, spaceId, logViewId } = action; + const { + datafeedSetupResults, + jobSetupResults, + jobSummaries, + spaceId, + logViewId, + idFormat, + } = action; const nextJobStatus = jobTypes.reduce( (accumulatedJobStatus, jobType) => ({ ...accumulatedJobStatus, [jobType]: - hasSuccessfullyCreatedJob(getJobId(spaceId, logViewId, jobType))(jobSetupResults) && - hasSuccessfullyStartedDatafeed(getDatafeedId(spaceId, logViewId, jobType))( + hasSuccessfullyCreatedJob(getJobId(spaceId, logViewId, idFormat, jobType))( + jobSetupResults + ) && + hasSuccessfullyStartedDatafeed(getDatafeedId(spaceId, logViewId, idFormat, jobType))( datafeedSetupResults ) ? 'started' : 'failed', }), - {} as Record + {} as Record ); const nextSetupStatus: SetupStatus = Object.values(nextJobStatus).every( (jobState) => jobState === 'started' || jobState === 'starting' @@ -129,7 +138,7 @@ const createStatusReducer = ...accumulatedJobStatus, [jobType]: 'failed', }), - {} as Record + {} as Record ), setupStatus: { type: 'failed', reasons: action.reason ? [action.reason] : ['unknown'] }, }; @@ -142,15 +151,15 @@ const createStatusReducer = }; } case 'fetchedJobStatuses': { - const { payload: jobSummaries, spaceId, logViewId } = action; + const { payload: jobSummaries, spaceId, logViewId, idFormat } = action; const { setupStatus } = state; const nextJobStatus = jobTypes.reduce( (accumulatedJobStatus, jobType) => ({ ...accumulatedJobStatus, - [jobType]: getJobStatus(getJobId(spaceId, logViewId, jobType))(jobSummaries), + [jobType]: getJobStatus(getJobId(spaceId, logViewId, idFormat, jobType))(jobSummaries), }), - {} as Record + {} as Record ); const nextSetupStatus = getSetupStatus(nextJobStatus)(setupStatus); @@ -170,7 +179,7 @@ const createStatusReducer = ...accumulatedJobStatus, [jobType]: 'unknown', }), - {} as Record + {} as Record ), }; } @@ -243,7 +252,7 @@ const getJobStatus = })[0] || 'missing'; const getSetupStatus = - (everyJobStatus: Record) => + (everyJobStatus: Record) => (previousSetupStatus: SetupStatus): SetupStatus => Object.entries(everyJobStatus).reduce((setupStatus, [, jobStatus]) => { if (jobStatus === 'missing') { @@ -265,6 +274,6 @@ const hasError = ( value: Value ): value is MandatoryProperty => value.error != null; -export const useModuleStatus = (jobTypes: JobType[]) => { +export const useModuleStatus = (jobTypes: T[]) => { return useReducer(createStatusReducer(jobTypes), { jobTypes }, createInitialState); }; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts index d6e95b7feebe5..30ca6f730bfb0 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { ValidateLogEntryDatasetsResponsePayload, ValidationIndicesResponsePayload, @@ -19,16 +20,17 @@ import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api'; export type { JobModelSizeStats, JobSummary } from './api/ml_get_jobs_summary_api'; -export interface ModuleDescriptor { +export interface ModuleDescriptor { moduleId: string; moduleName: string; moduleDescription: string; - jobTypes: JobType[]; + jobTypes: T[]; bucketSpan: number; - getJobIds: (spaceId: string, logViewId: string) => Record; + getJobIds: (spaceId: string, logViewId: string, idFormat: IdFormat) => Record; getJobSummary: ( spaceId: string, logViewId: string, + idFormat: IdFormat, fetch: HttpHandler ) => Promise; getModuleDefinition: (fetch: HttpHandler) => Promise; @@ -42,6 +44,7 @@ export interface ModuleDescriptor { cleanUpModule: ( spaceId: string, logViewId: string, + idFormat: IdFormat, fetch: HttpHandler ) => Promise; validateSetupIndices: ( diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts index fad6fd56f6251..f96e336080f10 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts @@ -8,6 +8,7 @@ import { isEqual } from 'lodash'; import { useCallback, useEffect, useMemo, useState } from 'react'; import usePrevious from 'react-use/lib/usePrevious'; +import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { combineDatasetFilters, DatasetFilter, @@ -30,21 +31,21 @@ type SetupHandler = ( datasetFilter: DatasetFilter ) => void; -interface AnalysisSetupStateArguments { +interface AnalysisSetupStateArguments { cleanUpAndSetUpModule: SetupHandler; - moduleDescriptor: ModuleDescriptor; + moduleDescriptor: ModuleDescriptor; setUpModule: SetupHandler; sourceConfiguration: ModuleSourceConfiguration; } const fourWeeksInMs = 86400000 * 7 * 4; -export const useAnalysisSetupState = ({ +export const useAnalysisSetupState = ({ cleanUpAndSetUpModule, moduleDescriptor: { validateSetupDatasets, validateSetupIndices }, setUpModule, sourceConfiguration, -}: AnalysisSetupStateArguments) => { +}: AnalysisSetupStateArguments) => { const { services } = useKibanaContextForPlugin(); const [startTime, setStartTime] = useState(Date.now() - fourWeeksInMs); const [endTime, setEndTime] = useState(undefined); diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts index 38a1c5c907528..ecfba10da19e6 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts @@ -8,11 +8,13 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { i18n } from '@kbn/i18n'; import type { HttpHandler } from '@kbn/core/public'; +import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; import { bucketSpan, categoriesMessageField, DatasetFilter, getJobId, + logEntryCategoriesJobType, LogEntryCategoriesJobType, logEntryCategoriesJobTypes, partitionField, @@ -36,21 +38,26 @@ const moduleDescription = i18n.translate( } ); -const getJobIds = (spaceId: string, logViewId: string) => +const getJobIds = (spaceId: string, logViewId: string, idFormat: IdFormat) => logEntryCategoriesJobTypes.reduce( (accumulatedJobIds, jobType) => ({ ...accumulatedJobIds, - [jobType]: getJobId(spaceId, logViewId, jobType), + [jobType]: getJobId(spaceId, logViewId, idFormat, jobType), }), {} as Record ); -const getJobSummary = async (spaceId: string, logViewId: string, fetch: HttpHandler) => { +const getJobSummary = async ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + fetch: HttpHandler +) => { const response = await callJobsSummaryAPI( - { spaceId, logViewId, jobTypes: logEntryCategoriesJobTypes }, + { spaceId, logViewId, idFormat, jobTypes: logEntryCategoriesJobTypes }, fetch ); - const jobIds = Object.values(getJobIds(spaceId, logViewId)); + const jobIds = Object.values(getJobIds(spaceId, logViewId, idFormat)); return response.filter((jobSummary) => jobIds.includes(jobSummary.id)); }; @@ -69,7 +76,7 @@ const setUpModule = async ( const indexNamePattern = indices.join(','); const jobOverrides = [ { - job_id: 'log-entry-categories-count' as const, + job_id: logEntryCategoriesJobType, analysis_config: { bucket_span: `${bucketSpan}ms`, }, @@ -88,7 +95,7 @@ const setUpModule = async ( ]; const datafeedOverrides = [ { - job_id: 'log-entry-categories-count' as const, + job_id: logEntryCategoriesJobType, runtime_mappings: runtimeMappings, }, ]; @@ -130,8 +137,19 @@ const setUpModule = async ( ); }; -const cleanUpModule = async (spaceId: string, logViewId: string, fetch: HttpHandler) => { - return await cleanUpJobsAndDatafeeds(spaceId, logViewId, logEntryCategoriesJobTypes, fetch); +const cleanUpModule = async ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + fetch: HttpHandler +) => { + return await cleanUpJobsAndDatafeeds( + spaceId, + logViewId, + idFormat, + logEntryCategoriesJobTypes, + fetch + ); }; const validateSetupIndices = async ( diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx index 3f4d802940974..a1c5192501970 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx @@ -8,6 +8,7 @@ import createContainer from 'constate'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { useMemo } from 'react'; +import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; import { useLogAnalysisModule } from '../../log_analysis_module'; import { useLogAnalysisModuleConfiguration } from '../../log_analysis_module_configuration'; import { useLogAnalysisModuleDefinition } from '../../log_analysis_module_definition'; @@ -19,12 +20,14 @@ export const useLogEntryCategoriesModule = ({ indexPattern, logViewId, spaceId, + idFormat, timestampField, runtimeMappings, }: { indexPattern: string; logViewId: string; spaceId: string; + idFormat: IdFormat; timestampField: string; runtimeMappings: estypes.MappingRuntimeFields; }) => { @@ -41,6 +44,7 @@ export const useLogEntryCategoriesModule = ({ const logAnalysisModule = useLogAnalysisModule({ moduleDescriptor: logEntryCategoriesModule, + idFormat, sourceConfiguration, }); @@ -51,6 +55,7 @@ export const useLogEntryCategoriesModule = ({ const { fetchModuleDefinition, getIsJobDefinitionOutdated } = useLogAnalysisModuleDefinition({ sourceConfiguration, + idFormat, moduleDescriptor: logEntryCategoriesModule, }); diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts index 8cb0bb8a21799..a51b18732be1d 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts @@ -8,10 +8,12 @@ import { i18n } from '@kbn/i18n'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { HttpHandler } from '@kbn/core/public'; +import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; import { bucketSpan, DatasetFilter, getJobId, + logEntryRateJobType, LogEntryRateJobType, logEntryRateJobTypes, partitionField, @@ -35,21 +37,26 @@ const moduleDescription = i18n.translate( } ); -const getJobIds = (spaceId: string, logViewId: string) => +const getJobIds = (spaceId: string, logViewId: string, idFormat: IdFormat) => logEntryRateJobTypes.reduce( (accumulatedJobIds, jobType) => ({ ...accumulatedJobIds, - [jobType]: getJobId(spaceId, logViewId, jobType), + [jobType]: getJobId(spaceId, logViewId, idFormat, jobType), }), {} as Record ); -const getJobSummary = async (spaceId: string, logViewId: string, fetch: HttpHandler) => { +const getJobSummary = async ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + fetch: HttpHandler +) => { const response = await callJobsSummaryAPI( - { spaceId, logViewId, jobTypes: logEntryRateJobTypes }, + { spaceId, logViewId, idFormat, jobTypes: logEntryRateJobTypes }, fetch ); - const jobIds = Object.values(getJobIds(spaceId, logViewId)); + const jobIds = Object.values(getJobIds(spaceId, logViewId, idFormat)); return response.filter((jobSummary) => jobIds.includes(jobSummary.id)); }; @@ -68,7 +75,7 @@ const setUpModule = async ( const indexNamePattern = indices.join(','); const jobOverrides = [ { - job_id: 'log-entry-rate' as const, + job_id: logEntryRateJobType, analysis_config: { bucket_span: `${bucketSpan}ms`, }, @@ -86,7 +93,7 @@ const setUpModule = async ( ]; const datafeedOverrides = [ { - job_id: 'log-entry-rate' as const, + job_id: logEntryRateJobType, runtime_mappings: runtimeMappings, }, ]; @@ -122,8 +129,13 @@ const setUpModule = async ( ); }; -const cleanUpModule = async (spaceId: string, logViewId: string, fetch: HttpHandler) => { - return await cleanUpJobsAndDatafeeds(spaceId, logViewId, logEntryRateJobTypes, fetch); +const cleanUpModule = async ( + spaceId: string, + logViewId: string, + idFormat: IdFormat, + fetch: HttpHandler +) => { + return await cleanUpJobsAndDatafeeds(spaceId, logViewId, idFormat, logEntryRateJobTypes, fetch); }; const validateSetupIndices = async ( diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx index 65bddee00ce30..ed5a977e38cbb 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx @@ -8,6 +8,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import createContainer from 'constate'; import { useMemo } from 'react'; +import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; import { ModuleSourceConfiguration } from '../../log_analysis_module_types'; import { useLogAnalysisModule } from '../../log_analysis_module'; import { useLogAnalysisModuleConfiguration } from '../../log_analysis_module_configuration'; @@ -18,12 +19,14 @@ export const useLogEntryRateModule = ({ indexPattern, logViewId, spaceId, + idFormat, timestampField, runtimeMappings, }: { indexPattern: string; logViewId: string; spaceId: string; + idFormat: IdFormat; timestampField: string; runtimeMappings: estypes.MappingRuntimeFields; }) => { @@ -40,6 +43,7 @@ export const useLogEntryRateModule = ({ const logAnalysisModule = useLogAnalysisModule({ moduleDescriptor: logEntryRateModule, + idFormat, sourceConfiguration, }); @@ -50,6 +54,7 @@ export const useLogEntryRateModule = ({ const { fetchModuleDefinition, getIsJobDefinitionOutdated } = useLogAnalysisModuleDefinition({ sourceConfiguration, + idFormat, moduleDescriptor: logEntryRateModule, }); diff --git a/x-pack/plugins/infra/public/containers/ml/api/ml_get_module.ts b/x-pack/plugins/infra/public/containers/ml/api/ml_get_module.ts index afb1ee211a75a..1cd85d448e7be 100644 --- a/x-pack/plugins/infra/public/containers/ml/api/ml_get_module.ts +++ b/x-pack/plugins/infra/public/containers/ml/api/ml_get_module.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import type { HttpHandler } from '@kbn/core/public'; -import { jobCustomSettingsRT } from '../../../../common/log_analysis'; +import { jobCustomSettingsRT } from '../../../../common/infra_ml'; import { decodeOrThrow } from '../../../../common/runtime_types'; export const callGetMlModuleAPI = async (moduleId: string, fetch: HttpHandler) => { diff --git a/x-pack/plugins/infra/public/containers/ml/infra_ml_module_definition.tsx b/x-pack/plugins/infra/public/containers/ml/infra_ml_module_definition.tsx index 88b7cfd941092..1dca20d586a50 100644 --- a/x-pack/plugins/infra/public/containers/ml/infra_ml_module_definition.tsx +++ b/x-pack/plugins/infra/public/containers/ml/infra_ml_module_definition.tsx @@ -6,7 +6,7 @@ */ import { useCallback, useMemo, useState } from 'react'; -import { getJobId } from '../../../common/log_analysis'; +import { getJobId } from '../../../common/infra_ml'; import { useKibanaContextForPlugin } from '../../hooks/use_kibana'; import { useTrackedPromise } from '../../utils/use_tracked_promise'; import { JobSummary } from './api/ml_get_jobs_summary_api'; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page.tsx index 34634b194cb85..f5b1e89c69e0b 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page.tsx @@ -11,6 +11,7 @@ import { useLogsBreadcrumbs } from '../../../hooks/use_logs_breadcrumbs'; import { LogEntryCategoriesPageContent } from './page_content'; import { LogEntryCategoriesPageProviders } from './page_providers'; import { logCategoriesTitle } from '../../../translations'; +import { LogMlJobIdFormatsShimProvider } from '../shared/use_log_ml_job_id_formats_shim'; export const LogEntryCategoriesPage = () => { useLogsBreadcrumbs([ @@ -21,9 +22,11 @@ export const LogEntryCategoriesPage = () => { return ( - - - + + + + + ); }; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx index d6a1e9f2ddc5e..c58ffc5f36e84 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx @@ -9,7 +9,7 @@ import { i18n } from '@kbn/i18n'; import React, { useCallback, useEffect } from 'react'; import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; -import { isJobStatusWithResults } from '../../../../common/log_analysis'; +import { isJobStatusWithResults, logEntryCategoriesJobType } from '../../../../common/log_analysis'; import { LoadingPage } from '../../../components/loading_page'; import { LogAnalysisSetupStatusUnknownPrompt, @@ -26,6 +26,7 @@ import { useLogEntryCategoriesModuleContext } from '../../../containers/logs/log import { LogsPageTemplate } from '../shared/page_template'; import { LogEntryCategoriesResultsContent } from './page_results_content'; import { LogEntryCategoriesSetupContent } from './page_setup_content'; +import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim'; const logCategoriesTitle = i18n.translate('xpack.infra.logs.logCategoriesTitle', { defaultMessage: 'Categories', @@ -52,6 +53,8 @@ export const LogEntryCategoriesPageContent = () => { } }, [fetchJobStatus, hasLogAnalysisReadCapabilities]); + const { idFormats } = useLogMlJobIdFormatsShimContext(); + if (!hasLogAnalysisCapabilites) { return ( { ); - } else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) { + } else if (isJobStatusWithResults(jobStatus[logEntryCategoriesJobType])) { return ( <> diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx index 5cb6a12166c53..74b041e466f55 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx @@ -7,12 +7,14 @@ import React from 'react'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; +import { logEntryCategoriesJobType } from '../../../../common/log_analysis'; import { InlineLogViewSplashPage } from '../../../components/logging/inline_log_view_splash_page'; import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout'; import { SourceLoadingPage } from '../../../components/source_loading_page'; import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories'; import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; import { ConnectedLogViewErrorPage } from '../shared/page_log_view_error'; +import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim'; export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ children }) => { const { @@ -25,6 +27,8 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child revertToDefaultLogView, } = useLogViewContext(); const { space } = useActiveKibanaSpace(); + const { idFormats, isLoadingLogAnalysisIdFormats, hasFailedLoadingLogAnalysisIdFormats } = + useLogMlJobIdFormatsShimContext(); // This is a rather crude way of guarding the dependent providers against // arguments that are only made available asynchronously. Ideally, we'd use @@ -33,9 +37,9 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child return null; } else if (!isPersistedLogView) { return ; - } else if (hasFailedLoading) { + } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) { return ; - } else if (isLoading || isUninitialized) { + } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats) { return ; } else if (resolvedLogView != null) { if (logViewReference.type === 'log-view-inline') { @@ -46,6 +50,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} + idFormat={idFormats![logEntryCategoriesJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx index e1e317136deed..c2ccd01b1632c 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx @@ -16,6 +16,7 @@ import { useKibana } from '@kbn/kibana-react-plugin/public'; import { MLJobsAwaitingNodeWarning, ML_PAGES, useMlHref } from '@kbn/ml-plugin/public'; import { useTrackPageview } from '@kbn/observability-shared-plugin/public'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; +import { logEntryCategoriesJobType } from '../../../../common/log_analysis'; import { TimeRange } from '../../../../common/time/time_range'; import { CategoryJobNoticesSection } from '../../../components/logging/log_analysis_job_status'; import { AnalyzeInMlButton } from '../../../components/logging/log_analysis_results'; @@ -33,17 +34,19 @@ import { StringTimeRange, useLogEntryCategoriesResultsUrlState, } from './use_log_entry_categories_results_url_state'; +import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; const JOB_STATUS_POLLING_INTERVAL = 30000; interface LogEntryCategoriesResultsContentProps { onOpenSetup: () => void; pageTitle: string; + idFormat: IdFormat; } export const LogEntryCategoriesResultsContent: React.FunctionComponent< LogEntryCategoriesResultsContentProps -> = ({ onOpenSetup, pageTitle }) => { +> = ({ onOpenSetup, pageTitle, idFormat }) => { useTrackPageview({ app: 'infra_logs', path: 'log_entry_categories_results' }); useTrackPageview({ app: 'infra_logs', path: 'log_entry_categories_results', delay: 15000 }); @@ -110,6 +113,7 @@ export const LogEntryCategoriesResultsContent: React.FunctionComponent< filteredDatasets: categoryQueryDatasets, onGetTopLogEntryCategoriesError: showLoadDataErrorNotification, logViewReference: { type: 'log-view-reference', logViewId }, + idFormat, startTime: categoryQueryTimeRange.timeRange.startTime, }); @@ -195,7 +199,7 @@ export const LogEntryCategoriesResultsContent: React.FunctionComponent< const analyzeInMlLink = useMlHref(ml, http.basePath.get(), { page: ML_PAGES.ANOMALY_EXPLORER, pageState: { - jobIds: [jobIds['log-entry-categories-count']], + jobIds: [jobIds[logEntryCategoriesJobType]], timeRange: { from: moment(categoryQueryTimeRange.timeRange.startTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), to: moment(categoryQueryTimeRange.timeRange.endTime).format('YYYY-MM-DDTHH:mm:ss.SSSZ'), @@ -264,7 +268,7 @@ export const LogEntryCategoriesResultsContent: React.FunctionComponent< = ({ categoryId, timeRange, logViewReference }) => { + const { idFormats } = useLogMlJobIdFormatsShimContext(); + const { getLogEntryCategoryExamples, hasFailedLoadingLogEntryCategoryExamples, @@ -29,6 +33,7 @@ export const CategoryDetailsRow: React.FunctionComponent<{ endTime: timeRange.endTime, exampleCount, logViewReference, + idFormat: idFormats![logEntryCategoriesJobType], startTime: timeRange.startTime, }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts index 14e7ebfbebd35..f37d1bfd68ebf 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getLogEntryCategoryDatasetsRequestPayloadRT, @@ -17,6 +18,7 @@ import { decodeOrThrow } from '../../../../../common/runtime_types'; interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; endTime: number; } @@ -25,7 +27,7 @@ export const callGetLogEntryCategoryDatasetsAPI = async ( requestArgs: RequestArgs, fetch: HttpHandler ) => { - const { logViewReference, startTime, endTime } = requestArgs; + const { logViewReference, idFormat, startTime, endTime } = requestArgs; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_DATASETS_PATH, { method: 'POST', @@ -33,6 +35,7 @@ export const callGetLogEntryCategoryDatasetsAPI = async ( getLogEntryCategoryDatasetsRequestPayloadRT.encode({ data: { logView: logViewReference, + idFormat, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts index 3e4947ca1e84f..18fdcfe75631a 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getLogEntryCategoryExamplesRequestPayloadRT, @@ -17,6 +18,7 @@ import { decodeOrThrow } from '../../../../../common/runtime_types'; interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; endTime: number; categoryId: number; @@ -27,7 +29,7 @@ export const callGetLogEntryCategoryExamplesAPI = async ( requestArgs: RequestArgs, fetch: HttpHandler ) => { - const { logViewReference, startTime, endTime, categoryId, exampleCount } = requestArgs; + const { logViewReference, idFormat, startTime, endTime, categoryId, exampleCount } = requestArgs; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORY_EXAMPLES_PATH, { method: 'POST', @@ -37,6 +39,7 @@ export const callGetLogEntryCategoryExamplesAPI = async ( categoryId, exampleCount, logView: logViewReference, + idFormat, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts index c4a1b6d095a29..8cdf92a5e6e6c 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getLogEntryCategoriesRequestPayloadRT, @@ -18,6 +19,7 @@ import { decodeOrThrow } from '../../../../../common/runtime_types'; interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; endTime: number; categoryCount: number; @@ -29,7 +31,8 @@ export const callGetTopLogEntryCategoriesAPI = async ( requestArgs: RequestArgs, fetch: HttpHandler ) => { - const { logViewReference, startTime, endTime, categoryCount, datasets, sort } = requestArgs; + const { logViewReference, idFormat, startTime, endTime, categoryCount, datasets, sort } = + requestArgs; const intervalDuration = endTime - startTime; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_CATEGORIES_PATH, { @@ -38,6 +41,7 @@ export const callGetTopLogEntryCategoriesAPI = async ( getLogEntryCategoriesRequestPayloadRT.encode({ data: { logView: logViewReference, + idFormat, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts index 20f7adb106857..979b8dcd6cdb0 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts @@ -8,6 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { GetLogEntryCategoriesSuccessResponsePayload, GetLogEntryCategoryDatasetsSuccessResponsePayload, @@ -32,6 +33,7 @@ export const useLogEntryCategoriesResults = ({ onGetLogEntryCategoryDatasetsError, onGetTopLogEntryCategoriesError, logViewReference, + idFormat, startTime, }: { categoriesCount: number; @@ -40,6 +42,7 @@ export const useLogEntryCategoriesResults = ({ onGetLogEntryCategoryDatasetsError?: (error: Error) => void; onGetTopLogEntryCategoriesError?: (error: Error) => void; logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; }) => { const [sortOptions, setSortOptions] = useState({ @@ -58,6 +61,7 @@ export const useLogEntryCategoriesResults = ({ return await callGetTopLogEntryCategoriesAPI( { logViewReference, + idFormat, startTime, endTime, categoryCount: categoriesCount, @@ -80,7 +84,15 @@ export const useLogEntryCategoriesResults = ({ } }, }, - [categoriesCount, endTime, filteredDatasets, logViewReference.logViewId, startTime, sortOptions] + [ + categoriesCount, + endTime, + filteredDatasets, + logViewReference.logViewId, + startTime, + sortOptions, + idFormat, + ] ); const [getLogEntryCategoryDatasetsRequest, getLogEntryCategoryDatasets] = useTrackedPromise( @@ -88,7 +100,7 @@ export const useLogEntryCategoriesResults = ({ cancelPreviousOn: 'creation', createPromise: async () => { return await callGetLogEntryCategoryDatasetsAPI( - { logViewReference, startTime, endTime }, + { logViewReference, idFormat, startTime, endTime }, services.http.fetch ); }, @@ -105,7 +117,7 @@ export const useLogEntryCategoriesResults = ({ } }, }, - [categoriesCount, endTime, logViewReference.logViewId, startTime] + [categoriesCount, endTime, logViewReference.logViewId, idFormat, startTime] ); const isLoadingTopLogEntryCategories = useMemo( diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx index c5516fdbc02f9..801c5da57ee0e 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx @@ -8,6 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { LogEntryCategoryExample } from '../../../../common/http_api'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; @@ -18,12 +19,14 @@ export const useLogEntryCategoryExamples = ({ endTime, exampleCount, logViewReference, + idFormat, startTime, }: { categoryId: number; endTime: number; exampleCount: number; logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; }) => { const { services } = useKibanaContextForPlugin(); @@ -39,6 +42,7 @@ export const useLogEntryCategoryExamples = ({ return await callGetLogEntryCategoryExamplesAPI( { logViewReference, + idFormat, startTime, endTime, categoryId, @@ -51,7 +55,7 @@ export const useLogEntryCategoryExamples = ({ setLogEntryCategoryExamples(examples); }, }, - [categoryId, endTime, exampleCount, logViewReference, startTime] + [categoryId, endTime, exampleCount, logViewReference, startTime, idFormat] ); const isLoadingLogEntryCategoryExamples = useMemo( diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page.tsx index 94950b24b1a94..50a4852c458c4 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page.tsx @@ -11,6 +11,7 @@ import { LogEntryRatePageContent } from './page_content'; import { LogEntryRatePageProviders } from './page_providers'; import { useLogsBreadcrumbs } from '../../../hooks/use_logs_breadcrumbs'; import { anomaliesTitle } from '../../../translations'; +import { LogMlJobIdFormatsShimProvider } from '../shared/use_log_ml_job_id_formats_shim'; export const LogEntryRatePage = () => { useLogsBreadcrumbs([ @@ -20,9 +21,11 @@ export const LogEntryRatePage = () => { ]); return ( - - - + + + + + ); }; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx index 533381dcbf7c3..e047408aec48c 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx @@ -10,7 +10,11 @@ import React, { memo, useCallback, useEffect } from 'react'; import useInterval from 'react-use/lib/useInterval'; import type { LazyObservabilityPageTemplateProps } from '@kbn/observability-shared-plugin/public'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; -import { isJobStatusWithResults } from '../../../../common/log_analysis'; +import { + isJobStatusWithResults, + logEntryCategoriesJobType, + logEntryRateJobType, +} from '../../../../common/log_analysis'; import { LoadingPage } from '../../../components/loading_page'; import { LogAnalysisSetupStatusUnknownPrompt, @@ -28,6 +32,7 @@ import { useLogEntryRateModuleContext } from '../../../containers/logs/log_analy import { LogsPageTemplate } from '../shared/page_template'; import { LogEntryRateResultsContent } from './page_results_content'; import { LogEntryRateSetupContent } from './page_setup_content'; +import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim'; const JOB_STATUS_POLLING_INTERVAL = 30000; @@ -89,6 +94,8 @@ export const LogEntryRatePageContent = memo(() => { } }, JOB_STATUS_POLLING_INTERVAL); + const { idFormats } = useLogMlJobIdFormatsShimContext(); + if (!hasLogAnalysisCapabilites) { return ( { ); } else if ( - isJobStatusWithResults(logEntryCategoriesJobStatus['log-entry-categories-count']) || - isJobStatusWithResults(logEntryRateJobStatus['log-entry-rate']) + isJobStatusWithResults(logEntryCategoriesJobStatus[logEntryCategoriesJobType]) || + isJobStatusWithResults(logEntryRateJobStatus[logEntryRateJobType]) ) { return ( <> - + ); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx index 46ce90cff63cc..e7ba080b6af02 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx @@ -7,6 +7,7 @@ import React from 'react'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; +import { logEntryCategoriesJobType, logEntryRateJobType } from '../../../../common/log_analysis'; import { InlineLogViewSplashPage } from '../../../components/logging/inline_log_view_splash_page'; import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout'; import { SourceLoadingPage } from '../../../components/source_loading_page'; @@ -15,6 +16,7 @@ import { LogEntryRateModuleProvider } from '../../../containers/logs/log_analysi import { LogEntryFlyoutProvider } from '../../../containers/logs/log_flyout'; import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; import { ConnectedLogViewErrorPage } from '../shared/page_log_view_error'; +import { useLogMlJobIdFormatsShimContext } from '../shared/use_log_ml_job_id_formats_shim'; export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) => { const { @@ -29,6 +31,9 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) const { space } = useActiveKibanaSpace(); + const { idFormats, isLoadingLogAnalysisIdFormats, hasFailedLoadingLogAnalysisIdFormats } = + useLogMlJobIdFormatsShimContext(); + // This is a rather crude way of guarding the dependent providers against // arguments that are only made available asynchronously. Ideally, we'd use // React concurrent mode and Suspense in order to handle that more gracefully. @@ -36,9 +41,9 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) return null; } else if (!isPersistedLogView) { return ; - } else if (isLoading || isUninitialized) { + } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats) { return ; - } else if (hasFailedLoading) { + } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) { return ; } else if (resolvedLogView != null) { if (logViewReference.type === 'log-view-inline') { @@ -50,6 +55,7 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} + idFormat={idFormats![logEntryRateJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > @@ -57,6 +63,7 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} + idFormat={idFormats![logEntryCategoriesJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx index a4d861e38ade1..d4564aaa1b182 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx @@ -15,7 +15,12 @@ import { useKibana } from '@kbn/kibana-react-plugin/public'; import { MLJobsAwaitingNodeWarning } from '@kbn/ml-plugin/public'; import { useTrackPageview } from '@kbn/observability-shared-plugin/public'; import { useLogViewContext, LogEntryFlyout } from '@kbn/logs-shared-plugin/public'; -import { isJobStatusWithResults } from '../../../../common/log_analysis'; +import { IdFormatByJobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { + isJobStatusWithResults, + logEntryCategoriesJobType, + logEntryRateJobType, +} from '../../../../common/log_analysis'; import { TimeKey } from '../../../../common/time'; import { CategoryJobNoticesSection, @@ -45,7 +50,8 @@ export const PAGINATION_DEFAULTS = { export const LogEntryRateResultsContent: React.FunctionComponent<{ pageTitle: string; -}> = ({ pageTitle }) => { + idFormats: IdFormatByJobType; +}> = ({ pageTitle, idFormats }) => { useTrackPageview({ app: 'infra_logs', path: 'log_entry_rate_results' }); useTrackPageview({ app: 'infra_logs', path: 'log_entry_rate_results', delay: 15000 }); @@ -82,11 +88,11 @@ export const LogEntryRateResultsContent: React.FunctionComponent<{ const jobIds = useMemo(() => { return [ - ...(isJobStatusWithResults(logEntryRateJobStatus['log-entry-rate']) - ? [logEntryRateJobIds['log-entry-rate']] + ...(isJobStatusWithResults(logEntryRateJobStatus[logEntryRateJobType]) + ? [logEntryRateJobIds[logEntryRateJobType]] : []), - ...(isJobStatusWithResults(logEntryCategoriesJobStatus['log-entry-categories-count']) - ? [logEntryCategoriesJobIds['log-entry-categories-count']] + ...(isJobStatusWithResults(logEntryCategoriesJobStatus[logEntryCategoriesJobType]) + ? [logEntryCategoriesJobIds[logEntryCategoriesJobType]] : []), ]; }, [ @@ -146,6 +152,7 @@ export const LogEntryRateResultsContent: React.FunctionComponent<{ isLoadingDatasets, } = useLogEntryAnomaliesResults({ logViewReference, + idFormats, startTime: timeRange.value.startTime, endTime: timeRange.value.endTime, defaultSortOptions: SORT_DEFAULTS, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx index 5fe150e3c2724..83e8184d34c93 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx @@ -12,11 +12,16 @@ import React from 'react'; import useMount from 'react-use/lib/useMount'; import { euiStyled } from '@kbn/kibana-react-plugin/common'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; -import { isCategoryAnomaly, LogEntryAnomaly } from '../../../../../../common/log_analysis'; +import { + isCategoryAnomaly, + LogEntryAnomaly, + logEntryRateJobType, +} from '../../../../../../common/log_analysis'; import { TimeRange } from '../../../../../../common/time/time_range'; import { LogEntryExampleMessages } from '../../../../../components/logging/log_entry_examples/log_entry_examples'; import { useLogEntryExamples } from '../../use_log_entry_examples'; import { LogEntryExampleMessage, LogEntryExampleMessageHeaders } from './log_entry_example'; +import { useLogMlJobIdFormatsShimContext } from '../../../shared/use_log_ml_job_id_formats_shim'; const EXAMPLE_COUNT = 5; @@ -29,6 +34,7 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{ timeRange: TimeRange; }> = ({ anomaly, timeRange }) => { const { logViewReference } = useLogViewContext(); + const { idFormats } = useLogMlJobIdFormatsShimContext(); if (logViewReference.type === 'log-view-inline') { throw new Error('Logs ML features only support persisted Log Views'); @@ -44,6 +50,7 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{ endTime: anomaly.startTime + anomaly.duration, exampleCount: EXAMPLE_COUNT, logViewReference, + idFormat: idFormats![logEntryRateJobType], startTime: anomaly.startTime, categoryId: isCategoryAnomaly(anomaly) ? anomaly.categoryId : undefined, }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts index b6a515ae6f134..19ffc3691540f 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormatByJobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getLogEntryAnomaliesRequestPayloadRT, getLogEntryAnomaliesSuccessReponsePayloadRT, @@ -17,6 +18,7 @@ import { AnomaliesSort, Pagination } from '../../../../../common/log_analysis'; interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormats: IdFormatByJobType; startTime: number; endTime: number; sort: AnomaliesSort; @@ -25,13 +27,15 @@ interface RequestArgs { } export const callGetLogEntryAnomaliesAPI = async (requestArgs: RequestArgs, fetch: HttpHandler) => { - const { logViewReference, startTime, endTime, sort, pagination, datasets } = requestArgs; + const { logViewReference, idFormats, startTime, endTime, sort, pagination, datasets } = + requestArgs; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_PATH, { method: 'POST', body: JSON.stringify( getLogEntryAnomaliesRequestPayloadRT.encode({ data: { logView: logViewReference, + idFormats, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts index a93712c5d5a86..1795fc50111f7 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormatByJobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { decodeOrThrow } from '../../../../../common/runtime_types'; import { getLogEntryAnomaliesDatasetsRequestPayloadRT, @@ -16,6 +17,7 @@ import { interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormats: IdFormatByJobType; startTime: number; endTime: number; } @@ -24,13 +26,14 @@ export const callGetLogEntryAnomaliesDatasetsAPI = async ( requestArgs: RequestArgs, fetch: HttpHandler ) => { - const { logViewReference, startTime, endTime } = requestArgs; + const { logViewReference, idFormats, startTime, endTime } = requestArgs; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_DATASETS_PATH, { method: 'POST', body: JSON.stringify( getLogEntryAnomaliesDatasetsRequestPayloadRT.encode({ data: { logView: logViewReference, + idFormats, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts index 6cf35b95868e1..36c0239ddeabb 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts @@ -7,6 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; import { getLogEntryExamplesRequestPayloadRT, @@ -17,6 +18,7 @@ import { decodeOrThrow } from '../../../../../common/runtime_types'; interface RequestArgs { logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; endTime: number; dataset: string; @@ -25,7 +27,8 @@ interface RequestArgs { } export const callGetLogEntryExamplesAPI = async (requestArgs: RequestArgs, fetch: HttpHandler) => { - const { logViewReference, startTime, endTime, dataset, exampleCount, categoryId } = requestArgs; + const { logViewReference, idFormat, startTime, endTime, dataset, exampleCount, categoryId } = + requestArgs; const response = await fetch(LOG_ANALYSIS_GET_LOG_ENTRY_RATE_EXAMPLES_PATH, { method: 'POST', body: JSON.stringify( @@ -34,6 +37,7 @@ export const callGetLogEntryExamplesAPI = async (requestArgs: RequestArgs, fetch dataset, exampleCount, logView: logViewReference, + idFormat, timeRange: { startTime, endTime, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts index 745b5a7cd5aec..420ee814996dd 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts @@ -8,6 +8,7 @@ import { useMemo, useState, useCallback, useEffect, useReducer } from 'react'; import useMount from 'react-use/lib/useMount'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormatByJobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { useTrackedPromise, CanceledPromiseError } from '../../../utils/use_tracked_promise'; import { callGetLogEntryAnomaliesAPI } from './service_calls/get_log_entry_anomalies'; import { callGetLogEntryAnomaliesDatasetsAPI } from './service_calls/get_log_entry_anomalies_datasets'; @@ -139,6 +140,7 @@ export const useLogEntryAnomaliesResults = ({ endTime, startTime, logViewReference, + idFormats, defaultSortOptions, defaultPaginationOptions, onGetLogEntryAnomaliesDatasetsError, @@ -147,6 +149,7 @@ export const useLogEntryAnomaliesResults = ({ endTime: number; startTime: number; logViewReference: PersistedLogViewReference; + idFormats: IdFormatByJobType; defaultSortOptions: AnomaliesSort; defaultPaginationOptions: Pick; onGetLogEntryAnomaliesDatasetsError?: (error: Error) => void; @@ -185,6 +188,7 @@ export const useLogEntryAnomaliesResults = ({ return await callGetLogEntryAnomaliesAPI( { logViewReference, + idFormats, startTime: queryStartTime, endTime: queryEndTime, sort: sortOptions, @@ -218,6 +222,7 @@ export const useLogEntryAnomaliesResults = ({ }, [ logViewReference, + idFormats, dispatch, reducerState.timeRange, reducerState.sortOptions, @@ -295,7 +300,7 @@ export const useLogEntryAnomaliesResults = ({ cancelPreviousOn: 'creation', createPromise: async () => { return await callGetLogEntryAnomaliesDatasetsAPI( - { logViewReference, startTime, endTime }, + { logViewReference, idFormats, startTime, endTime }, services.http.fetch ); }, @@ -312,7 +317,7 @@ export const useLogEntryAnomaliesResults = ({ } }, }, - [endTime, logViewReference, startTime] + [endTime, logViewReference, idFormats, startTime] ); const isLoadingDatasets = useMemo( diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts index 4301f08ab41da..ca247664cb807 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts @@ -8,6 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; import { LogEntryExample } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; @@ -18,6 +19,7 @@ export const useLogEntryExamples = ({ endTime, exampleCount, logViewReference, + idFormat, startTime, categoryId, }: { @@ -25,6 +27,7 @@ export const useLogEntryExamples = ({ endTime: number; exampleCount: number; logViewReference: PersistedLogViewReference; + idFormat: IdFormat; startTime: number; categoryId?: string; }) => { @@ -38,6 +41,7 @@ export const useLogEntryExamples = ({ return await callGetLogEntryExamplesAPI( { logViewReference, + idFormat, startTime, endTime, dataset, @@ -51,7 +55,7 @@ export const useLogEntryExamples = ({ setLogEntryExamples(examples); }, }, - [dataset, endTime, exampleCount, logViewReference, startTime] + [dataset, endTime, exampleCount, logViewReference, startTime, idFormat] ); const isLoadingLogEntryExamples = useMemo( diff --git a/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts b/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts new file mode 100644 index 0000000000000..e13ab2408f6b5 --- /dev/null +++ b/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { HttpHandler } from '@kbn/core/public'; +import { decodeOrThrow } from '../../../../common/runtime_types'; +import { + getLogAnalysisIdFormatsRequestPayloadRT, + getLogAnalysisIdFormatsSuccessResponsePayloadRT, + LOG_ANALYSIS_GET_ID_FORMATS, +} from '../../../../common/http_api/log_analysis/v1/id_formats'; + +interface RequestArgs { + spaceId: string; + logViewId: string; +} + +export const callGetLogAnalysisIdFormats = async (requestArgs: RequestArgs, fetch: HttpHandler) => { + const { logViewId, spaceId } = requestArgs; + const response = await fetch(LOG_ANALYSIS_GET_ID_FORMATS, { + method: 'POST', + body: JSON.stringify( + getLogAnalysisIdFormatsRequestPayloadRT.encode({ + data: { + logViewId, + spaceId, + }, + }) + ), + version: '1', + }); + + return decodeOrThrow(getLogAnalysisIdFormatsSuccessResponsePayloadRT)(response); +}; diff --git a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx new file mode 100644 index 0000000000000..1a7e1bdc27ebc --- /dev/null +++ b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import createContainer from 'constate'; +import { useState, useEffect, useMemo, useCallback } from 'react'; +import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; +import { IdFormatByJobType, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; +import { useTrackedPromise } from '../../../utils/use_tracked_promise'; + +import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; +import { callGetLogAnalysisIdFormats } from './call_get_log_analysis_id_formats'; + +const useLogMlJobIdFormatsShim = () => { + const [idFormats, setIdFormats] = useState(null); + + const { logViewReference } = useLogViewContext(); + const { space } = useActiveKibanaSpace(); + const { services } = useKibanaContextForPlugin(); + + const [getLogAnalysisIdFormatsRequest, getLogAnalysisIdFormats] = useTrackedPromise( + { + cancelPreviousOn: 'creation', + createPromise: async () => { + if (!space) { + return { data: null }; + } + + if (logViewReference.type === 'log-view-inline') { + throw new Error('Logs ML features only support persisted Log Views'); + } + + return await callGetLogAnalysisIdFormats( + { + logViewId: logViewReference.logViewId, + spaceId: space.id, + }, + services.http.fetch + ); + }, + onResolve: ({ data }) => { + setIdFormats(data); + }, + }, + [logViewReference, space] + ); + + useEffect(() => { + getLogAnalysisIdFormats(); + }, [getLogAnalysisIdFormats]); + + const isLoadingLogAnalysisIdFormats = useMemo( + () => getLogAnalysisIdFormatsRequest.state === 'pending', + [getLogAnalysisIdFormatsRequest.state] + ); + + const hasFailedLoadingLogAnalysisIdFormats = useMemo( + () => getLogAnalysisIdFormatsRequest.state === 'rejected', + [getLogAnalysisIdFormatsRequest.state] + ); + + const migrateIdFormat = useCallback( + (jobType: JobType) => { + setIdFormats({ + ...idFormats!, + [jobType]: 'hashed', + }); + }, + [idFormats] + ); + + return { + idFormats, + migrateIdFormat, + isLoadingLogAnalysisIdFormats, + hasFailedLoadingLogAnalysisIdFormats, + }; +}; + +export const [LogMlJobIdFormatsShimProvider, useLogMlJobIdFormatsShimContext] = + createContainer(useLogMlJobIdFormatsShim); diff --git a/x-pack/plugins/infra/server/infra_server.ts b/x-pack/plugins/infra/server/infra_server.ts index e54713faba76a..8bbef44d39557 100644 --- a/x-pack/plugins/infra/server/infra_server.ts +++ b/x-pack/plugins/infra/server/infra_server.ts @@ -21,6 +21,7 @@ import { initGetLogEntryExamplesRoute, initValidateLogAnalysisDatasetsRoute, initValidateLogAnalysisIndicesRoute, + initGetLogAnalysisIdFormatsRoute, } from './routes/log_analysis'; import { initMetadataRoute } from './routes/metadata'; import { initMetricsAPIRoute } from './routes/metrics_api'; @@ -46,6 +47,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => { initSnapshotRoute(libs); initNodeDetailsRoute(libs); initMetricsSourceConfigurationRoutes(libs); + initGetLogAnalysisIdFormatsRoute(libs); initValidateLogAnalysisDatasetsRoute(libs); initValidateLogAnalysisIndicesRoute(libs); initGetLogEntryExamplesRoute(libs); diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts index 591376450be38..23fc34dd5f7fc 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts @@ -7,13 +7,16 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { PersistedLogViewReference, ResolvedLogView } from '@kbn/logs-shared-plugin/common'; +import { IdFormat, IdFormatByJobType } from '../../../common/http_api/log_analysis/v1/id_formats'; import { AnomaliesSort, getJobId, isCategoryAnomaly, jobCustomSettingsRT, LogEntryAnomalyDatasets, + logEntryCategoriesJobType, logEntryCategoriesJobTypes, + logEntryRateJobType, logEntryRateJobTypes, Pagination, } from '../../../common/log_analysis'; @@ -55,10 +58,21 @@ interface MappedAnomalyHit { async function getCompatibleAnomaliesJobIds( spaceId: string, logViewId: string, + idFormats: IdFormatByJobType, mlAnomalyDetectors: MlAnomalyDetectors ) { - const logRateJobId = getJobId(spaceId, logViewId, logEntryRateJobTypes[0]); - const logCategoriesJobId = getJobId(spaceId, logViewId, logEntryCategoriesJobTypes[0]); + const logRateJobId = getJobId( + spaceId, + logViewId, + idFormats[logEntryRateJobType], + logEntryRateJobType + ); + const logCategoriesJobId = getJobId( + spaceId, + logViewId, + idFormats[logEntryCategoriesJobType], + logEntryCategoriesJobType + ); const jobIds: string[] = []; let jobSpans: TracingSpan[] = []; @@ -100,6 +114,7 @@ export async function getLogEntryAnomalies( infra: Promise>; }, logView: PersistedLogViewReference, + idFormats: IdFormatByJobType, startTime: number, endTime: number, sort: AnomaliesSort, @@ -115,6 +130,7 @@ export async function getLogEntryAnomalies( } = await getCompatibleAnomaliesJobIds( infraContext.spaceId, logView.logViewId, + idFormats, infraContext.mlAnomalyDetectors ); @@ -156,7 +172,8 @@ export async function getLogEntryAnomalies( const logEntryCategoriesCountJobId = getJobId( infraContext.spaceId, logView.logViewId, - logEntryCategoriesJobTypes[0] + idFormats[logEntryCategoriesJobType], + logEntryCategoriesJobType ); const { logEntryCategoriesById } = await fetchLogEntryCategories( @@ -332,6 +349,7 @@ export async function getLogEntryExamples( infra: Promise>; }, logView: PersistedLogViewReference, + idFormat: IdFormat, startTime: number, endTime: number, dataset: string, @@ -346,6 +364,7 @@ export async function getLogEntryExamples( const jobId = getJobId( infraContext.spaceId, logView.logViewId, + idFormat, categoryId != null ? logEntryCategoriesJobTypes[0] : logEntryRateJobTypes[0] ); @@ -371,6 +390,7 @@ export async function getLogEntryExamples( } = await fetchLogEntryExamples( context, logView, + idFormat, indices, runtimeMappings, timestampField, @@ -398,6 +418,7 @@ export async function fetchLogEntryExamples( infra: Promise>; }, logView: PersistedLogViewReference, + idFormat: IdFormat, indices: string, runtimeMappings: estypes.MappingRuntimeFields, timestampField: string, @@ -421,6 +442,7 @@ export async function fetchLogEntryExamples( const logEntryCategoriesCountJobId = getJobId( infraContext.spaceId, logView.logViewId, + idFormat, logEntryCategoriesJobTypes[0] ); @@ -484,6 +506,7 @@ export async function getLogEntryAnomaliesDatasets( }; }, logView: PersistedLogViewReference, + idFormats: IdFormatByJobType, startTime: number, endTime: number ) { @@ -493,6 +516,7 @@ export async function getLogEntryAnomaliesDatasets( } = await getCompatibleAnomaliesJobIds( context.infra.spaceId, logView.logViewId, + idFormats, context.infra.mlAnomalyDetectors ); diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts index 88678f4c79c53..c92443b07c401 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts @@ -12,6 +12,7 @@ import { PersistedLogViewReference, ResolvedLogView, } from '@kbn/logs-shared-plugin/common'; +import { IdFormat } from '../../../common/http_api/log_analysis/v1/id_formats'; import { CategoriesSort, compareDatasetsByMaximumAnomalyScore, @@ -51,6 +52,7 @@ export async function getTopLogEntryCategories( }; }, logView: PersistedLogViewReference, + idFormat: IdFormat, startTime: number, endTime: number, categoryCount: number, @@ -63,6 +65,7 @@ export async function getTopLogEntryCategories( const logEntryCategoriesCountJobId = getJobId( context.infra.spaceId, logView.logViewId, + idFormat, logEntryCategoriesJobTypes[0] ); @@ -123,12 +126,14 @@ export async function getLogEntryCategoryDatasets( }; }, logView: PersistedLogViewReference, + idFormat: IdFormat, startTime: number, endTime: number ) { const logEntryCategoriesCountJobId = getJobId( context.infra.spaceId, logView.logViewId, + idFormat, logEntryCategoriesJobTypes[0] ); @@ -147,6 +152,7 @@ export async function getLogEntryCategoryExamples( }; }, logView: PersistedLogViewReference, + idFormat: IdFormat, startTime: number, endTime: number, categoryId: number, @@ -158,6 +164,7 @@ export async function getLogEntryCategoryExamples( const logEntryCategoriesCountJobId = getJobId( context.infra.spaceId, logView.logViewId, + idFormat, logEntryCategoriesJobTypes[0] ); diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts index 1e043fed0986a..a7df06e25bd98 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { IdFormat } from '../../../common/http_api/log_analysis/v1/id_formats'; import { decodeOrThrow } from '../../../common/runtime_types'; import { logRateModelPlotResponseRT, @@ -12,7 +13,7 @@ import { LogRateModelPlotBucket, CompositeTimestampPartitionKey, } from './queries'; -import { getJobId } from '../../../common/log_analysis'; +import { getJobId, logEntryRateJobType } from '../../../common/log_analysis'; import type { MlSystem } from '../../types'; const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000; @@ -25,12 +26,13 @@ export async function getLogEntryRateBuckets( }; }, logViewId: string, + idFormat: IdFormat, startTime: number, endTime: number, bucketDuration: number, datasets?: string[] ) { - const logRateJobId = getJobId(context.infra.spaceId, logViewId, 'log-entry-rate'); + const logRateJobId = getJobId(context.infra.spaceId, logViewId, idFormat, logEntryRateJobType); let mlModelPlotBuckets: LogRateModelPlotBucket[] = []; let afterLatestBatchKey: CompositeTimestampPartitionKey | undefined; diff --git a/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts b/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts new file mode 100644 index 0000000000000..c4a1d61e5d694 --- /dev/null +++ b/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MlAnomalyDetectors } from '@kbn/ml-plugin/server'; +import { + IdFormat, + IdFormatByJobType, + JobType, +} from '../../../common/http_api/log_analysis/v1/id_formats'; +import { + getJobId, + logEntryCategoriesJobType, + logEntryRateJobType, +} from '../../../common/log_analysis'; + +export async function resolveIdFormats( + spaceId: string, + logViewId: string, + mlAnomalyDetectors: MlAnomalyDetectors +): Promise { + const entryRateFormat = await resolveIdFormat( + spaceId, + logViewId, + logEntryRateJobType, + mlAnomalyDetectors + ); + const entryCategoriesCountFormat = await resolveIdFormat( + spaceId, + logViewId, + logEntryCategoriesJobType, + mlAnomalyDetectors + ); + + return { + [logEntryRateJobType]: entryRateFormat, + [logEntryCategoriesJobType]: entryCategoriesCountFormat, + }; +} + +async function resolveIdFormat( + spaceId: string, + logViewId: string, + jobType: JobType, + mlAnomalyDetectors: MlAnomalyDetectors +): Promise { + try { + const hashedJobId = getJobId(spaceId, logViewId, 'hashed', jobType); + const hashedJobs = await mlAnomalyDetectors.jobs(hashedJobId); + if (hashedJobs.count > 0) { + return 'hashed'; + } + } catch (e) { + // Ignore 404 in case the job isn't found + if (e.statusCode !== 404) { + throw e; + } + } + + try { + const legacyJobId = getJobId(spaceId, logViewId, 'legacy', jobType); + const legacyJobs = await mlAnomalyDetectors.jobs(legacyJobId); + if (legacyJobs.count > 0) { + return 'legacy'; + } + } catch (e) { + // Ignore 404 in case the job isn't found + if (e.statusCode !== 404) { + throw e; + } + } + + return 'hashed'; +} diff --git a/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts b/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts new file mode 100644 index 0000000000000..7802b630af98d --- /dev/null +++ b/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import Boom from '@hapi/boom'; +import { createValidationFunction } from '@kbn/logs-shared-plugin/common/runtime_types'; +import * as logAnalysisIdFormatsV1 from '../../../common/http_api/log_analysis/v1/id_formats'; +import { InfraBackendLibs } from '../../lib/infra_types'; +import { isMlPrivilegesError } from '../../lib/log_analysis'; +import { resolveIdFormats } from '../../lib/log_analysis/resolve_id_formats'; +import { assertHasInfraMlPlugins } from '../../utils/request_context'; + +export const initGetLogAnalysisIdFormatsRoute = ({ framework }: InfraBackendLibs) => { + framework + .registerVersionedRoute({ + access: 'internal', + method: 'post', + path: logAnalysisIdFormatsV1.LOG_ANALYSIS_GET_ID_FORMATS, + }) + .addVersion( + { + version: '1', + validate: { + request: { + body: createValidationFunction( + logAnalysisIdFormatsV1.getLogAnalysisIdFormatsRequestPayloadRT + ), + }, + }, + }, + framework.router.handleLegacyErrors(async (requestContext, request, response) => { + const { + data: { logViewId, spaceId }, + } = request.body; + + try { + const infraMlContext = await assertHasInfraMlPlugins(requestContext); + const mlAnomalyDetectors = (await infraMlContext.infra).mlAnomalyDetectors; + + const idFormatByJobType = await resolveIdFormats(logViewId, spaceId, mlAnomalyDetectors); + + return response.ok({ + body: logAnalysisIdFormatsV1.getLogAnalysisIdFormatsSuccessResponsePayloadRT.encode({ + data: idFormatByJobType, + }), + }); + } catch (error) { + if (Boom.isBoom(error)) { + throw error; + } + + if (isMlPrivilegesError(error)) { + return response.customError({ + statusCode: 403, + body: { + message: error.message, + }, + }); + } + + return response.customError({ + statusCode: error.statusCode ?? 500, + body: { + message: error.message ?? 'An unexpected error occurred', + }, + }); + } + }) + ); +}; diff --git a/x-pack/plugins/infra/server/routes/log_analysis/index.ts b/x-pack/plugins/infra/server/routes/log_analysis/index.ts index a642cd830b6fb..a3266adfd6ddd 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/index.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/index.ts @@ -7,3 +7,4 @@ export * from './results'; export * from './validation'; +export { initGetLogAnalysisIdFormatsRoute } from './id_formats'; diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies.ts index 30a9aadda432a..1af2b1e7f9806 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies.ts @@ -40,6 +40,7 @@ export const initGetLogEntryAnomaliesRoute = ({ framework }: InfraBackendLibs) = const { data: { logView, + idFormats, timeRange: { startTime, endTime }, sort: sortParam, pagination: paginationParam, @@ -60,6 +61,7 @@ export const initGetLogEntryAnomaliesRoute = ({ framework }: InfraBackendLibs) = } = await getLogEntryAnomalies( infraMlContext, logView, + idFormats, startTime, endTime, sort, diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies_datasets.ts index ce10ba09a059f..0b6444c1a9d7b 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies_datasets.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_anomalies_datasets.ts @@ -39,6 +39,7 @@ export const initGetLogEntryAnomaliesDatasetsRoute = ({ framework }: InfraBacken const { data: { logView, + idFormats, timeRange: { startTime, endTime }, }, } = request.body; @@ -49,6 +50,7 @@ export const initGetLogEntryAnomaliesDatasetsRoute = ({ framework }: InfraBacken const { datasets, timing } = await getLogEntryAnomaliesDatasets( { infra: await infraMlContext.infra }, logView, + idFormats, startTime, endTime ); diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_categories.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_categories.ts index f51f81a846bbb..92221d5ce359d 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_categories.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_categories.ts @@ -41,6 +41,7 @@ export const initGetLogEntryCategoriesRoute = ({ framework }: InfraBackendLibs) categoryCount, histograms, logView, + idFormat, timeRange: { startTime, endTime }, datasets, sort, @@ -53,6 +54,7 @@ export const initGetLogEntryCategoriesRoute = ({ framework }: InfraBackendLibs) const { data: topLogEntryCategories, timing } = await getTopLogEntryCategories( { infra: await infraMlContext.infra }, logView, + idFormat, startTime, endTime, categoryCount, diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets.ts index 9ed89f1adb05b..1c62b67091a56 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets.ts @@ -39,6 +39,7 @@ export const initGetLogEntryCategoryDatasetsRoute = ({ framework }: InfraBackend const { data: { logView, + idFormat, timeRange: { startTime, endTime }, }, } = request.body; @@ -49,6 +50,7 @@ export const initGetLogEntryCategoryDatasetsRoute = ({ framework }: InfraBackend const { data: logEntryCategoryDatasets, timing } = await getLogEntryCategoryDatasets( { infra: await infraMlContext.infra }, logView, + idFormat, startTime, endTime ); diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_examples.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_examples.ts index 828912143d412..c80aed6eab0fb 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_examples.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_examples.ts @@ -44,6 +44,7 @@ export const initGetLogEntryCategoryExamplesRoute = ({ categoryId, exampleCount, logView, + idFormat, timeRange: { startTime, endTime }, }, } = request.body; @@ -59,6 +60,7 @@ export const initGetLogEntryCategoryExamplesRoute = ({ const { data: logEntryCategoryExamples, timing } = await getLogEntryCategoryExamples( { infra: await infraMlContext.infra, core: await infraMlContext.core }, logView, + idFormat, startTime, endTime, categoryId, diff --git a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_examples.ts b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_examples.ts index df79783a56edc..8be303ca01f8d 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_examples.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_examples.ts @@ -44,6 +44,7 @@ export const initGetLogEntryExamplesRoute = ({ dataset, exampleCount, logView, + idFormat, timeRange: { startTime, endTime }, categoryId, }, @@ -60,6 +61,7 @@ export const initGetLogEntryExamplesRoute = ({ const { data: logEntryExamples, timing } = await getLogEntryExamples( infraMlContext, logView, + idFormat, startTime, endTime, dataset, From 5deea41b7c0a06e66ab64d7ca61f6ab367148664 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Mon, 9 Oct 2023 15:09:56 +0200 Subject: [PATCH 02/11] Use uuid.v5 instead of crypto, update description of created jobs --- .../infra/common/log_analysis/job_parameters.ts | 11 +++++------ .../modules/log_entry_rate/module_descriptor.ts | 1 + 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts index 223304d171a4e..948492ccef7d4 100644 --- a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { createHash } from 'crypto'; +import { v5 } from 'uuid'; import { IdFormat, JobType } from '../http_api/log_analysis/v1/id_formats'; export const bucketSpan = 900000; @@ -15,15 +15,14 @@ export const categoriesMessageField = 'message'; export const partitionField = 'event.dataset'; +const ID_NAMESPACE = 'f91b78c0-fdd3-425d-a4ba-4c028fe57e0f'; + export const getJobIdPrefix = (spaceId: string, sourceId: string, idFormat: IdFormat) => { if (idFormat === 'legacy') { return `kibana-logs-ui-${spaceId}-${sourceId}-`; } else { - const hash = createHash('sha256') - .update(`${spaceId}-${sourceId}`) - .digest('hex') - .substring(0, 10); - return `logs-${hash}-`; + const uuid = v5(`${spaceId}-${sourceId}`, ID_NAMESPACE).replaceAll('-', ''); + return `logs-${uuid}-`; } }; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts index a51b18732be1d..7101e8ec99465 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts @@ -75,6 +75,7 @@ const setUpModule = async ( const indexNamePattern = indices.join(','); const jobOverrides = [ { + description: `Logs UI (${spaceId}/${sourceId}): Detects anomalies in the log entry ingestion rate`, job_id: logEntryRateJobType, analysis_config: { bucket_span: `${bucketSpan}ms`, From 000750c6c6198e829712ed9c136fc553e6eb0097 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 11 Oct 2023 21:35:56 +0200 Subject: [PATCH 03/11] Add basic tests and test helpers --- .../log_entry_rate/page_results_content.tsx | 1 + x-pack/test/functional/apps/infra/index.ts | 1 + .../ml_job_id_formats/assert_id_formats.ts | 46 ++++++++ .../infra/ml_job_id_formats/ml_job_configs.ts | 85 +++++++++++++++ .../infra/ml_job_id_formats/ml_job_helper.ts | 25 +++++ .../ml_job_id_formats/request_tracker.ts | 101 ++++++++++++++++++ .../apps/infra/ml_job_id_formats/tests.ts | 81 ++++++++++++++ .../services/logs_ui/log_entry_rate.ts | 4 + 8 files changed, 344 insertions(+) create mode 100644 x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts create mode 100644 x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts create mode 100644 x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts create mode 100644 x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts create mode 100644 x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx index d4564aaa1b182..d258e707094b2 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx @@ -206,6 +206,7 @@ export const LogEntryRateResultsContent: React.FunctionComponent<{ return ( { loadTestFile(require.resolve('./logs_source_configuration')); loadTestFile(require.resolve('./link_to')); loadTestFile(require.resolve('./log_stream')); + loadTestFile(require.resolve('./ml_job_id_formats/tests')); }); }); }; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts new file mode 100644 index 0000000000000..9a90dcd07eb50 --- /dev/null +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { IdFormat } from '@kbn/infra-plugin/common/http_api/log_analysis/v1/id_formats'; + +const rateHashedPattern = /logs-[0-9a-fA-F]{32,}-log-entry-rate/; +const rateLegacyPattern = /kibana-logs-ui-.*-.*-log-entry-rate/; +const categoriesCountHashedPattern = /logs-[0-9a-fA-F]{32,}-log-entry-categories-count/; +const categoriesCountLegacyPattern = /kibana-logs-ui-.*-.*-log-entry-categories-count/; + +export function assertIdFormats( + url: string, + expected: { + 'log-entry-rate': IdFormat | undefined; + 'log-entry-categories-count': IdFormat | undefined; + } +) { + const idFormats = extractIdFormats(url); + expect(idFormats).to.eql(expected); +} + +function extractIdFormats(url: string) { + let rateFormat; + if (rateHashedPattern.test(url)) { + rateFormat = 'hashed'; + } else if (rateLegacyPattern.test(url)) { + rateFormat = 'legacy'; + } + + let categoriesCountFormat; + if (categoriesCountHashedPattern.test(url)) { + categoriesCountFormat = 'hashed'; + } else if (categoriesCountLegacyPattern.test(url)) { + categoriesCountFormat = 'legacy'; + } + + return { + 'log-entry-rate': rateFormat, + 'log-entry-categories-count': categoriesCountFormat, + }; +} diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts new file mode 100644 index 0000000000000..abd40de4075a3 --- /dev/null +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { TypeOf } from '@kbn/config-schema'; +import { setupModuleBodySchema } from '@kbn/ml-plugin/server/routes/schemas/modules'; + +export interface MlJob { + jobId: string; + config: TypeOf; +} + +const baseConfig = { + start: Date.now(), + indexPatternName: 'filebeat-*', + startDatafeed: true, + jobOverrides: [ + { + job_id: '', + analysis_config: { + bucket_span: '900000ms', + }, + data_description: { + time_field: '@timestamp', + }, + custom_settings: { + logs_source_config: { + indexPattern: 'filebeat-*', + timestampField: '@timestamp', + bucketSpan: 900000, + }, + }, + }, + ], + datafeedOverrides: [ + { + job_id: '', + runtime_mappings: {}, + }, + ], + useDedicatedIndex: true, +}; + +const rateJobConfig = Object.assign({}, baseConfig); +rateJobConfig.jobOverrides[0].job_id = 'log-entry-rate'; +rateJobConfig.datafeedOverrides[0].job_id = 'log-entry-rate'; + +const categoriesCountConfig = Object.assign({}, baseConfig); +categoriesCountConfig.jobOverrides[0].job_id = 'log-entry-categories-count'; +categoriesCountConfig.datafeedOverrides[0].job_id = 'log-entry-categories-count'; + +export const legacyRateJob: MlJob = { + jobId: 'kibana-logs-ui-default-default-log-entry-rate', + config: { + ...rateJobConfig, + prefix: 'kibana-logs-ui-default-default-', + }, +}; + +export const hashedRateJob: MlJob = { + jobId: 'logs-11558ee526445db2b42eb3d6b4af58d0-log-entry-rate', + config: { + ...rateJobConfig, + prefix: 'logs-11558ee526445db2b42eb3d6b4af58d0-', + }, +}; + +export const legacyCategoriesCountJob: MlJob = { + jobId: 'kibana-logs-ui-default-default-log-entry-categories-count', + config: { + ...categoriesCountConfig, + prefix: 'kibana-logs-ui-default-default-', + }, +}; + +export const hashedCategoriesCountJob: MlJob = { + jobId: 'logs-11558ee526445db2b42eb3d6b4af58d0-categories-count', + config: { + ...categoriesCountConfig, + prefix: 'logs-11558ee526445db2b42eb3d6b4af58d0-', + }, +}; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts new file mode 100644 index 0000000000000..f89b3d064e62d --- /dev/null +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MlApi } from '../../../services/ml/api'; +import { MlJob } from './ml_job_configs'; + +export function createMlJobHelper(ml: MlApi) { + async function createMlJob(mlJob: MlJob) { + await ml.setupModule('logs_ui_analysis', mlJob.config, 'default'); + await ml.waitForAnomalyDetectionJobToExist(mlJob.jobId); + } + + async function deleteMlJob(mlJob: MlJob) { + await ml.deleteAnomalyDetectionJobES(mlJob.jobId); + await ml.waitForAnomalyDetectionJobNotToExist(mlJob.jobId); + } + + return { createMlJob, deleteMlJob }; +} + +export type MlJobHelper = ReturnType; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts new file mode 100644 index 0000000000000..ae7905e342a6c --- /dev/null +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Browser } from '../../../../../../test/functional/services/common/browser'; +import { CommonPageObject } from '../../../../../../test/functional/page_objects/common_page'; + +type PerformanceResourceTimingWithHttpStatus = PerformanceResourceTiming & { + responseStatus: number; +}; + +export interface RequestLogEntry { + url: string; + timestamp: number; + status: number; +} + +declare global { + interface Window { + ftrLogsUiAnomalies?: { + requests: RequestLogEntry[]; + observer: PerformanceObserver; + }; + } +} + +export function createRequestTracker(browser: Browser, common: CommonPageObject) { + async function install() { + await browser.execute(() => { + function handleTimings(entryList: PerformanceObserverEntryList) { + const entries = entryList.getEntriesByType( + 'resource' + ) as PerformanceResourceTimingWithHttpStatus[]; + + entries + .filter((entry) => entry.initiatorType === 'fetch') + .forEach((entry) => { + if (window.ftrLogsUiAnomalies) { + window.ftrLogsUiAnomalies.requests.push({ + url: entry.name, + timestamp: entry.startTime, + status: entry.responseStatus, + }); + } else { + throw new Error('Request tracker not installed'); + } + }); + } + + const observer = new PerformanceObserver(handleTimings); + observer.observe({ type: 'resource', buffered: true }); + + window.ftrLogsUiAnomalies = { + observer, + requests: [], + }; + }); + } + + async function getRequests(pattern: RegExp, timeToWait: number = 0) { + if (timeToWait > 0) { + await common.sleep(timeToWait); + } + + // Passing RegExp to the browser doesn't seem to serialize well + // so we pass a string, but .toString returns it like /pattern/ which + // when we compile it in the browser gets escaped to /\/pattern\// + // thus we remove the surrounding slashes + const patternString = pattern.toString(); + const trimmedPattern = patternString.substring(1, patternString.length - 1); + + return await browser.execute((browserPattern: string) => { + const regExp = new RegExp(browserPattern); + if (window.ftrLogsUiAnomalies) { + const entries = window.ftrLogsUiAnomalies.requests.filter((entry) => + regExp.test(entry.url) + ); + entries.sort((a, b) => a.timestamp - b.timestamp); + return entries; + } else { + throw new Error('Request tracker not installed'); + } + }, trimmedPattern); + } + + async function uninstall() { + await browser.execute(() => { + if (window.ftrLogsUiAnomalies) { + window.ftrLogsUiAnomalies.observer.disconnect(); + delete window.ftrLogsUiAnomalies; + } else { + throw new Error('Request tracker not installed'); + } + }); + } + + return { install, getRequests, uninstall }; +} diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts new file mode 100644 index 0000000000000..28fc87aae7844 --- /dev/null +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { assertIdFormats } from './assert_id_formats'; +import { createMlJobHelper, MlJobHelper } from './ml_job_helper'; +import { createRequestTracker } from './request_tracker'; +import { legacyRateJob, hashedRateJob } from './ml_job_configs'; + +export default ({ getService, getPageObjects }: FtrProviderContext) => { + const logsUi = getService('logsUi'); + const retry = getService('retry'); + const esArchiver = getService('esArchiver'); + const ml = getService('ml'); + const browser = getService('browser'); + const pageObjects = getPageObjects(['common']); + const requestTracker = createRequestTracker(browser, pageObjects.common); + let mlJobHelper: MlJobHelper; + + describe('ML job ID formats', function () { + this.tags('includeFirefox'); + + this.beforeAll(async () => { + // Access to ml.api has to happen inside a test or test hook + mlJobHelper = createMlJobHelper(ml.api); + }); + + it('loads jobs in the legacy ID format', async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/simple_logs'); + await mlJobHelper.createMlJob(legacyRateJob); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'legacy', + 'log-entry-categories-count': undefined, + }); + + await esArchiver.unload('x-pack/test/functional/es_archives/infra/simple_logs'); + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJob(legacyRateJob); + }); + + it('loads jobs in the hashed ID format', async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/simple_logs'); + await mlJobHelper.createMlJob(hashedRateJob); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': undefined, + }); + + await esArchiver.unload('x-pack/test/functional/es_archives/infra/simple_logs'); + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJob(hashedRateJob); + }); + }); +}; diff --git a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts index 6be84edeb1940..6564856ac3572 100644 --- a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts +++ b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts @@ -21,6 +21,10 @@ export function LogEntryRatePageProvider({ getPageObjects, getService }: FtrProv return await testSubjects.find('logEntryRateSetupPage'); }, + async getResultsScreen(): Promise { + return await testSubjects.find('logEntryRateResultsPage'); + }, + async getNoDataScreen() { return await testSubjects.find('noDataPage'); }, From dd029b6a96ddf9c431d8b1db5dabb0d5cf04ef22 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Thu, 12 Oct 2023 15:26:49 +0200 Subject: [PATCH 04/11] Add more tests --- .../setup_flyout/setup_flyout.tsx | 7 +- .../infra/ml_job_id_formats/ml_job_configs.ts | 90 +++-- .../infra/ml_job_id_formats/ml_job_helper.ts | 18 +- .../apps/infra/ml_job_id_formats/tests.ts | 319 +++++++++++++++--- .../services/logs_ui/log_entry_rate.ts | 36 ++ 5 files changed, 400 insertions(+), 70 deletions(-) diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx index 18bab7011eca8..33dce47b6f3cf 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx @@ -34,7 +34,12 @@ export const LogAnalysisSetupFlyout: React.FC<{ } return ( - +

diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts index abd40de4075a3..facd1a059d3f5 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts @@ -10,16 +10,19 @@ import { setupModuleBodySchema } from '@kbn/ml-plugin/server/routes/schemas/modu export interface MlJob { jobId: string; + module: 'logs_ui_analysis' | 'logs_ui_categories'; config: TypeOf; } -const baseConfig = { +const rateConfig = { + prefix: '', start: Date.now(), indexPatternName: 'filebeat-*', startDatafeed: true, + useDedicatedIndex: true, jobOverrides: [ { - job_id: '', + job_id: 'log-entry-rate', analysis_config: { bucket_span: '900000ms', }, @@ -37,49 +40,90 @@ const baseConfig = { ], datafeedOverrides: [ { - job_id: '', + job_id: 'log-entry-rate', runtime_mappings: {}, }, ], - useDedicatedIndex: true, }; -const rateJobConfig = Object.assign({}, baseConfig); -rateJobConfig.jobOverrides[0].job_id = 'log-entry-rate'; -rateJobConfig.datafeedOverrides[0].job_id = 'log-entry-rate'; - -const categoriesCountConfig = Object.assign({}, baseConfig); -categoriesCountConfig.jobOverrides[0].job_id = 'log-entry-categories-count'; -categoriesCountConfig.datafeedOverrides[0].job_id = 'log-entry-categories-count'; - -export const legacyRateJob: MlJob = { - jobId: 'kibana-logs-ui-default-default-log-entry-rate', - config: { - ...rateJobConfig, - prefix: 'kibana-logs-ui-default-default-', +const categoriesCountConfig = { + prefix: '', + start: Date.now(), + indexPatternName: 'filebeat-*', + startDatafeed: true, + useDedicatedIndex: true, + jobOverrides: [ + { + job_id: 'log-entry-categories-count', + analysis_config: { + bucket_span: '900000ms', + }, + data_description: { + time_field: '@timestamp', + }, + custom_settings: { + logs_source_config: { + indexPattern: 'filebeat-*', + timestampField: '@timestamp', + bucketSpan: 900000, + datasetFilter: { + type: 'includeAll', + }, + }, + }, + }, + ], + datafeedOverrides: [ + { + job_id: 'log-entry-categories-count', + runtime_mappings: {}, + }, + ], + query: { + bool: { + filter: [ + { + exists: { + field: 'message', + }, + }, + ], + }, }, }; export const hashedRateJob: MlJob = { jobId: 'logs-11558ee526445db2b42eb3d6b4af58d0-log-entry-rate', + module: 'logs_ui_analysis', config: { - ...rateJobConfig, + ...rateConfig, prefix: 'logs-11558ee526445db2b42eb3d6b4af58d0-', }, }; -export const legacyCategoriesCountJob: MlJob = { - jobId: 'kibana-logs-ui-default-default-log-entry-categories-count', +export const hashedCategoriesCountJob: MlJob = { + jobId: 'logs-11558ee526445db2b42eb3d6b4af58d0-log-entry-categories-count', + module: 'logs_ui_categories', config: { ...categoriesCountConfig, + prefix: 'logs-11558ee526445db2b42eb3d6b4af58d0-', + }, +}; + +export const legacyRateJob: MlJob = { + jobId: 'kibana-logs-ui-default-default-log-entry-rate', + module: 'logs_ui_analysis', + config: { + ...rateConfig, prefix: 'kibana-logs-ui-default-default-', }, }; -export const hashedCategoriesCountJob: MlJob = { - jobId: 'logs-11558ee526445db2b42eb3d6b4af58d0-categories-count', +export const legacyCategoriesCountJob: MlJob = { + jobId: 'kibana-logs-ui-default-default-log-entry-categories-count', + module: 'logs_ui_categories', config: { ...categoriesCountConfig, - prefix: 'logs-11558ee526445db2b42eb3d6b4af58d0-', + prefix: 'kibana-logs-ui-default-default-', }, }; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts index f89b3d064e62d..3f4f85d47f339 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts @@ -9,17 +9,21 @@ import { MlApi } from '../../../services/ml/api'; import { MlJob } from './ml_job_configs'; export function createMlJobHelper(ml: MlApi) { - async function createMlJob(mlJob: MlJob) { - await ml.setupModule('logs_ui_analysis', mlJob.config, 'default'); - await ml.waitForAnomalyDetectionJobToExist(mlJob.jobId); + async function createMlJobs(jobs: MlJob[]) { + for (const mlJob of jobs) { + await ml.setupModule(mlJob.module, mlJob.config, 'default'); + await ml.waitForAnomalyDetectionJobToExist(mlJob.jobId); + } } - async function deleteMlJob(mlJob: MlJob) { - await ml.deleteAnomalyDetectionJobES(mlJob.jobId); - await ml.waitForAnomalyDetectionJobNotToExist(mlJob.jobId); + async function deleteMlJobs(jobs: MlJob[]) { + for (const mlJob of jobs) { + await ml.deleteAnomalyDetectionJobES(mlJob.jobId); + await ml.waitForAnomalyDetectionJobNotToExist(mlJob.jobId); + } } - return { createMlJob, deleteMlJob }; + return { createMlJobs, deleteMlJobs }; } export type MlJobHelper = ReturnType; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts index 28fc87aae7844..9895a49fbcd9e 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts @@ -10,15 +10,20 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; import { assertIdFormats } from './assert_id_formats'; import { createMlJobHelper, MlJobHelper } from './ml_job_helper'; import { createRequestTracker } from './request_tracker'; -import { legacyRateJob, hashedRateJob } from './ml_job_configs'; +import { + hashedRateJob, + hashedCategoriesCountJob, + legacyRateJob, + legacyCategoriesCountJob, +} from './ml_job_configs'; export default ({ getService, getPageObjects }: FtrProviderContext) => { - const logsUi = getService('logsUi'); const retry = getService('retry'); const esArchiver = getService('esArchiver'); - const ml = getService('ml'); const browser = getService('browser'); const pageObjects = getPageObjects(['common']); + const logsUi = getService('logsUi'); + const ml = getService('ml'); const requestTracker = createRequestTracker(browser, pageObjects.common); let mlJobHelper: MlJobHelper; @@ -28,54 +33,290 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { this.beforeAll(async () => { // Access to ml.api has to happen inside a test or test hook mlJobHelper = createMlJobHelper(ml.api); + await esArchiver.load('x-pack/test/functional/es_archives/infra/simple_logs'); }); - it('loads jobs in the legacy ID format', async () => { - await esArchiver.load('x-pack/test/functional/es_archives/infra/simple_logs'); - await mlJobHelper.createMlJob(legacyRateJob); - await logsUi.logEntryRatePage.navigateTo(); - await requestTracker.install(); + this.afterAll(async () => { + await esArchiver.unload('x-pack/test/functional/es_archives/infra/simple_logs'); + }); - await retry.try(async () => { - expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + describe('hashed format', () => { + // The hashed format always takes priority. If, for some reason, the same job exists + // in both formats, only the hashed format job will be used. + + it('loads rate job in the hashed ID format', async () => { + await mlJobHelper.createMlJobs([hashedRateJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': undefined, + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedRateJob]); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); - expect(requests).not.to.be.empty(); - assertIdFormats(requests[0].url, { - 'log-entry-rate': 'legacy', - 'log-entry-categories-count': undefined, + + it('loads category count job in the hashed ID format', async () => { + await mlJobHelper.createMlJobs([hashedCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': undefined, + 'log-entry-categories-count': 'hashed', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedCategoriesCountJob]); }); - await esArchiver.unload('x-pack/test/functional/es_archives/infra/simple_logs'); - await requestTracker.uninstall(); - await mlJobHelper.deleteMlJob(legacyRateJob); + it('loads rate and category count job in the hashed ID format', async () => { + await mlJobHelper.createMlJobs([hashedRateJob, hashedCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': 'hashed', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedRateJob, hashedCategoriesCountJob]); + }); }); - it('loads jobs in the hashed ID format', async () => { - await esArchiver.load('x-pack/test/functional/es_archives/infra/simple_logs'); - await mlJobHelper.createMlJob(hashedRateJob); - await logsUi.logEntryRatePage.navigateTo(); - await requestTracker.install(); + describe('legacy format', () => { + it('loads rate job in the legacy ID format', async () => { + await mlJobHelper.createMlJobs([legacyRateJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'legacy', + 'log-entry-categories-count': undefined, + }); - await retry.try(async () => { - expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([legacyRateJob]); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); - expect(requests).not.to.be.empty(); - assertIdFormats(requests[0].url, { - 'log-entry-rate': 'hashed', - 'log-entry-categories-count': undefined, + + it('loads category count job in the legacy ID format', async () => { + await mlJobHelper.createMlJobs([legacyCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': undefined, + 'log-entry-categories-count': 'legacy', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([legacyCategoriesCountJob]); }); - await esArchiver.unload('x-pack/test/functional/es_archives/infra/simple_logs'); - await requestTracker.uninstall(); - await mlJobHelper.deleteMlJob(hashedRateJob); + it('loads rate and category count job in the legacy ID format', async () => { + await mlJobHelper.createMlJobs([legacyRateJob, legacyCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'legacy', + 'log-entry-categories-count': 'legacy', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([legacyRateJob, legacyCategoriesCountJob]); + }); + }); + + describe('mixed formats', () => { + it('loads rate job in the hashed format and category count job in the legacy format', async () => { + await mlJobHelper.createMlJobs([hashedRateJob, legacyCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': 'legacy', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedRateJob, legacyCategoriesCountJob]); + }); + + it('loads rate job in the legacy format and category count job in the hashed format', async () => { + await mlJobHelper.createMlJobs([legacyRateJob, hashedCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + assertIdFormats(requests[0].url, { + 'log-entry-rate': 'legacy', + 'log-entry-categories-count': 'hashed', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([legacyRateJob, hashedCategoriesCountJob]); + }); + }); + + describe('creation and recreation', () => { + it('create first ML job', async () => { + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getSetupScreen()).to.be.ok(); + }); + + await logsUi.logEntryRatePage.startJobSetup(); + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getSetupFlyout()).to.be.ok(); + }); + + await logsUi.logEntryRatePage.startRateJobCreation(); + await retry.waitFor( + 'Create ML job button is enabled', + async () => await logsUi.logEntryRatePage.canCreateJob() + ); + + await logsUi.logEntryRatePage.createJob(); + await retry.waitFor( + 'ML job created', + async () => await logsUi.logEntryRatePage.jobCreationDone() + ); + + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + + assertIdFormats(requests[requests.length - 1].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': undefined, + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedRateJob]); + }); + + it('create second ML job', async () => { + await mlJobHelper.createMlJobs([legacyRateJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); + + await logsUi.logEntryRatePage.manageMlJobs(); + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getSetupFlyout()).to.be.ok(); + }); + + await logsUi.logEntryRatePage.startCategoriesCountJobCreation(); + await retry.waitFor( + 'Create ML job button is enabled', + async () => await logsUi.logEntryRatePage.canCreateJob() + ); + + await logsUi.logEntryRatePage.createJob(); + await retry.waitFor( + 'ML job created', + async () => await logsUi.logEntryRatePage.jobCreationDone() + ); + + const requests = await requestTracker.getRequests( + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, + 2000 + ); + expect(requests).not.to.be.empty(); + + assertIdFormats(requests[requests.length - 1].url, { + 'log-entry-rate': 'legacy', + 'log-entry-categories-count': 'hashed', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([legacyRateJob, hashedCategoriesCountJob]); + }); + + // it('recreate legacy format job', () => { + // // create one legacy and one hashed job + // // try to recreate the legacy job and verify that it swaps the format to use + // }); + + // it('recreate hashed format job', () => { + // // create one legacy and one hashed job + // // try to recreate the hashed job and verify that the calls are made but no format changes are made + // }); }); }); }; diff --git a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts index 6564856ac3572..004dbe57a0f63 100644 --- a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts +++ b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts @@ -28,5 +28,41 @@ export function LogEntryRatePageProvider({ getPageObjects, getService }: FtrProv async getNoDataScreen() { return await testSubjects.find('noDataPage'); }, + + async startJobSetup() { + await testSubjects.click('infraLogEntryRateSetupContentMlSetupButton'); + }, + + async manageMlJobs() { + await testSubjects.click('infraManageJobsButtonManageMlJobsButton'); + }, + + async getSetupFlyout(): Promise { + return await testSubjects.find('infraLogAnalysisSetupFlyout'); + }, + + async startRateJobCreation() { + const buttons = await testSubjects.findAll('infraCreateJobButtonButton'); + await buttons[0].click(); + }, + + async startCategoriesCountJobCreation() { + const buttons = await testSubjects.findAll('infraCreateJobButtonButton'); + await buttons[1].click(); + }, + + async canCreateJob() { + const createJobButton = await testSubjects.find('infraCreateMLJobsButtonCreateMlJobButton'); + const disabled = await createJobButton.getAttribute('disabled'); + return disabled !== 'true'; + }, + + async createJob() { + await testSubjects.click('infraCreateMLJobsButtonCreateMlJobButton'); + }, + + async jobCreationDone() { + return await testSubjects.exists('infraProcessStepViewResultsButton'); + }, }; } From ae4de5ae2b678b3b84b24d96080cea0a97354629 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Thu, 12 Oct 2023 22:11:58 +0200 Subject: [PATCH 05/11] Add more tests --- .../apps/infra/ml_job_id_formats/tests.ts | 97 ++++++++++--------- .../services/logs_ui/log_entry_rate.ts | 12 +++ 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts index 9895a49fbcd9e..4625eee74383b 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts @@ -17,6 +17,9 @@ import { legacyCategoriesCountJob, } from './ml_job_configs'; +const anomalyDetectorsPattern = + /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/; + export default ({ getService, getPageObjects }: FtrProviderContext) => { const retry = getService('retry'); const esArchiver = getService('esArchiver'); @@ -52,10 +55,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { @@ -75,10 +75,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': undefined, @@ -97,10 +94,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': 'hashed', @@ -121,10 +115,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': 'legacy', @@ -143,10 +134,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': undefined, @@ -165,10 +153,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': 'legacy', @@ -189,10 +174,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': 'hashed', @@ -211,10 +193,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await retry.try(async () => { expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); }); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[0].url, { 'log-entry-rate': 'legacy', @@ -252,10 +231,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { async () => await logsUi.logEntryRatePage.jobCreationDone() ); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[requests.length - 1].url, { @@ -293,10 +269,7 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { async () => await logsUi.logEntryRatePage.jobCreationDone() ); - const requests = await requestTracker.getRequests( - /anomaly_detectors\/.*-log-entry-(rate|categories-count)\/results\/overall_buckets/, - 2000 - ); + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); expect(requests).not.to.be.empty(); assertIdFormats(requests[requests.length - 1].url, { @@ -308,15 +281,43 @@ export default ({ getService, getPageObjects }: FtrProviderContext) => { await mlJobHelper.deleteMlJobs([legacyRateJob, hashedCategoriesCountJob]); }); - // it('recreate legacy format job', () => { - // // create one legacy and one hashed job - // // try to recreate the legacy job and verify that it swaps the format to use - // }); + it('migrate legacy job', async () => { + await mlJobHelper.createMlJobs([legacyRateJob, hashedCategoriesCountJob]); + await logsUi.logEntryRatePage.navigateTo(); + await requestTracker.install(); + + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getResultsScreen()).to.be.ok(); + }); - // it('recreate hashed format job', () => { - // // create one legacy and one hashed job - // // try to recreate the hashed job and verify that the calls are made but no format changes are made - // }); + await logsUi.logEntryRatePage.manageMlJobs(); + await retry.try(async () => { + expect(await logsUi.logEntryRatePage.getSetupFlyout()).to.be.ok(); + }); + + await logsUi.logEntryRatePage.startRateJobCreation(); + await retry.waitFor( + 'Recreate ML job button is enabled', + async () => await logsUi.logEntryRatePage.canRecreateJob() + ); + + await logsUi.logEntryRatePage.recreateJob(); + await retry.waitFor( + 'ML job recreated', + async () => await logsUi.logEntryRatePage.jobCreationDone() + ); + + const requests = await requestTracker.getRequests(anomalyDetectorsPattern, 2000); + expect(requests).not.to.be.empty(); + + assertIdFormats(requests[requests.length - 1].url, { + 'log-entry-rate': 'hashed', + 'log-entry-categories-count': 'hashed', + }); + + await requestTracker.uninstall(); + await mlJobHelper.deleteMlJobs([hashedRateJob, hashedCategoriesCountJob]); + }); }); }); }; diff --git a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts index 004dbe57a0f63..bf58d74a06c44 100644 --- a/x-pack/test/functional/services/logs_ui/log_entry_rate.ts +++ b/x-pack/test/functional/services/logs_ui/log_entry_rate.ts @@ -61,6 +61,18 @@ export function LogEntryRatePageProvider({ getPageObjects, getService }: FtrProv await testSubjects.click('infraCreateMLJobsButtonCreateMlJobButton'); }, + async canRecreateJob() { + const createJobButton = await testSubjects.find( + 'infraRecreateMLJobsButtonRecreateMlJobsButton' + ); + const disabled = await createJobButton.getAttribute('disabled'); + return disabled !== 'true'; + }, + + async recreateJob() { + await testSubjects.click('infraRecreateMLJobsButtonRecreateMlJobsButton'); + }, + async jobCreationDone() { return await testSubjects.exists('infraProcessStepViewResultsButton'); }, From 9c5000ecc8adc5956d94ab06a5a45c67f410c414 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Fri, 13 Oct 2023 11:42:49 +0200 Subject: [PATCH 06/11] Add comment about removing dashes from UUID --- x-pack/plugins/infra/common/log_analysis/job_parameters.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts index 948492ccef7d4..499d8653f2166 100644 --- a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts @@ -21,6 +21,8 @@ export const getJobIdPrefix = (spaceId: string, sourceId: string, idFormat: IdFo if (idFormat === 'legacy') { return `kibana-logs-ui-${spaceId}-${sourceId}-`; } else { + // A UUID is 36 characters but based on the ML job names for logs, our limit is 32 characters + // Thus we remove the 4 dashes const uuid = v5(`${spaceId}-${sourceId}`, ID_NAMESPACE).replaceAll('-', ''); return `logs-${uuid}-`; } From afad134c8f424265faf7b6b99b0972b653dff144 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Mon, 13 Nov 2023 15:30:56 +0100 Subject: [PATCH 07/11] Change import to refernece http_api/latest --- x-pack/plugins/infra/common/http_api/latest.ts | 1 + .../log_analysis/{ => id_formats}/v1/id_formats.ts | 2 +- .../log_analysis/results/v1/log_entry_anomalies.ts | 2 +- .../results/v1/log_entry_anomalies_datasets.ts | 2 +- .../results/v1/log_entry_categories.ts | 2 +- .../results/v1/log_entry_category_datasets.ts | 2 +- .../results/v1/log_entry_category_examples.ts | 2 +- .../log_analysis/results/v1/log_entry_examples.ts | 2 +- .../infra/common/log_analysis/job_parameters.ts | 2 +- .../containers/logs/log_analysis/api/ml_cleanup.ts | 2 +- .../log_analysis/api/ml_get_jobs_summary_api.ts | 2 +- .../logs/log_analysis/log_analysis_cleanup.tsx | 2 +- .../logs/log_analysis/log_analysis_module.tsx | 3 +-- .../log_analysis_module_configuration.ts | 2 +- .../log_analysis_module_definition.tsx | 2 +- .../log_analysis/log_analysis_module_status.tsx | 2 +- .../logs/log_analysis/log_analysis_module_types.ts | 2 +- .../logs/log_analysis/log_analysis_setup_state.ts | 2 +- .../log_entry_categories/module_descriptor.ts | 2 +- .../use_log_entry_categories_module.tsx | 2 +- .../modules/log_entry_rate/module_descriptor.ts | 2 +- .../log_entry_rate/use_log_entry_rate_module.tsx | 2 +- .../log_entry_categories/page_results_content.tsx | 2 +- .../get_log_entry_category_datasets.ts | 2 +- .../get_log_entry_category_examples.ts | 2 +- .../service_calls/get_top_log_entry_categories.ts | 2 +- .../use_log_entry_categories_results.ts | 2 +- .../use_log_entry_category_examples.tsx | 2 +- .../logs/log_entry_rate/page_results_content.tsx | 2 +- .../service_calls/get_log_entry_anomalies.ts | 2 +- .../get_log_entry_anomalies_datasets.ts | 2 +- .../service_calls/get_log_entry_examples.ts | 2 +- .../use_log_entry_anomalies_results.ts | 2 +- .../logs/log_entry_rate/use_log_entry_examples.ts | 2 +- .../shared/call_get_log_analysis_id_formats.ts | 2 +- .../logs/shared/use_log_ml_job_id_formats_shim.tsx | 2 +- .../server/lib/log_analysis/log_entry_anomalies.ts | 2 +- .../log_analysis/log_entry_categories_analysis.ts | 2 +- .../lib/log_analysis/log_entry_rate_analysis.ts | 2 +- .../server/lib/log_analysis/resolve_id_formats.ts | 6 +----- .../infra/server/routes/log_analysis/id_formats.ts | 14 ++++++++------ .../infra/ml_job_id_formats/assert_id_formats.ts | 2 +- 42 files changed, 49 insertions(+), 51 deletions(-) rename x-pack/plugins/infra/common/http_api/log_analysis/{ => id_formats}/v1/id_formats.ts (97%) diff --git a/x-pack/plugins/infra/common/http_api/latest.ts b/x-pack/plugins/infra/common/http_api/latest.ts index 98787a2c581a2..9b9ba2a0c54f4 100644 --- a/x-pack/plugins/infra/common/http_api/latest.ts +++ b/x-pack/plugins/infra/common/http_api/latest.ts @@ -10,3 +10,4 @@ export * from './log_alerts/v1'; export * from './log_analysis/results/v1'; export * from './log_analysis/validation/v1'; export * from './metrics_explorer_views/v1'; +export * from './log_analysis/id_formats/v1/id_formats'; diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts b/x-pack/plugins/infra/common/http_api/log_analysis/id_formats/v1/id_formats.ts similarity index 97% rename from x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts rename to x-pack/plugins/infra/common/http_api/log_analysis/id_formats/v1/id_formats.ts index 1e323d44d0096..c148df6c4c506 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/v1/id_formats.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/id_formats/v1/id_formats.ts @@ -6,7 +6,7 @@ */ import * as rt from 'io-ts'; -import { logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT } from '../../../log_analysis'; +import { logEntryRateJobTypeRT, logEntryCategoriesJobTypeRT } from '../../../../log_analysis'; export const idFormatRT = rt.union([rt.literal('legacy'), rt.literal('hashed')]); export type IdFormat = rt.TypeOf; diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts index d5463185991c4..38244c6a869b7 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; -import { idFormatByJobTypeRT } from '../../v1/id_formats'; +import { idFormatByJobTypeRT } from '../../id_formats/v1/id_formats'; import { timeRangeRT, routeTimingMetadataRT } from '../../../shared'; import { logEntryAnomalyRT, diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts index 22460385d7c02..5b6031ce27587 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_anomalies_datasets.ts @@ -14,7 +14,7 @@ import { timeRangeRT, routeTimingMetadataRT, } from '../../../shared'; -import { idFormatByJobTypeRT } from '../../v1/id_formats'; +import { idFormatByJobTypeRT } from '../../id_formats/v1/id_formats'; export const LOG_ANALYSIS_GET_LOG_ENTRY_ANOMALIES_DATASETS_PATH = '/api/infra/log_analysis/results/log_entry_anomalies_datasets'; diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts index 0e0f2f74e6e38..525292fb46ee3 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_categories.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; -import { idFormatRT } from '../../v1/id_formats'; +import { idFormatRT } from '../../id_formats/v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts index 48d4b8c9f8ae7..5b258d05e6cc9 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_datasets.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; -import { idFormatRT } from '../../v1/id_formats'; +import { idFormatRT } from '../../id_formats/v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts index 8eb4baef59f13..c0b7b3c00b551 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_category_examples.ts @@ -7,7 +7,7 @@ import { logEntryContextRT, persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; import * as rt from 'io-ts'; -import { idFormatRT } from '../../v1/id_formats'; +import { idFormatRT } from '../../id_formats/v1/id_formats'; import { badRequestErrorRT, forbiddenErrorRT, diff --git a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts index 5f61d0be72ace..4a0779a9128f8 100644 --- a/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts +++ b/x-pack/plugins/infra/common/http_api/log_analysis/results/v1/log_entry_examples.ts @@ -7,7 +7,7 @@ import * as rt from 'io-ts'; import { persistedLogViewReferenceRT } from '@kbn/logs-shared-plugin/common'; -import { idFormatRT } from '../../v1/id_formats'; +import { idFormatRT } from '../../id_formats/v1/id_formats'; import { logEntryExampleRT } from '../../../../log_analysis'; import { badRequestErrorRT, diff --git a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts index 499d8653f2166..22fcdea971e23 100644 --- a/x-pack/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/plugins/infra/common/log_analysis/job_parameters.ts @@ -7,7 +7,7 @@ import * as rt from 'io-ts'; import { v5 } from 'uuid'; -import { IdFormat, JobType } from '../http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../http_api/latest'; export const bucketSpan = 900000; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts index 8509951e9daf2..b27938f795eea 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import type { HttpHandler } from '@kbn/core/public'; -import { IdFormat, JobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../../common/http_api/latest'; import { getDatafeedId, getJobId } from '../../../../../common/log_analysis'; import { decodeOrThrow } from '../../../../../common/runtime_types'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts index a684eba1ac9b7..9e2996215df8d 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts @@ -8,7 +8,7 @@ import * as rt from 'io-ts'; import type { HttpHandler } from '@kbn/core/public'; -import { IdFormat, JobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../../common/http_api/latest'; import { getJobId, jobCustomSettingsRT } from '../../../../../common/log_analysis'; import { decodeOrThrow } from '../../../../../common/runtime_types'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx index b8aa4a6f810da..b70ca688f53d6 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_cleanup.tsx @@ -6,7 +6,7 @@ */ import type { HttpHandler } from '@kbn/core/public'; -import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../common/http_api/latest'; import { getJobId } from '../../../../common/log_analysis'; import { callDeleteJobs, callGetJobDeletionTasks, callStopDatafeeds } from './api/ml_cleanup'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx index a0618cc20231a..25ddd466e6e8f 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module.tsx @@ -8,8 +8,7 @@ import { useCallback, useMemo } from 'react'; import { useUiTracker } from '@kbn/observability-shared-plugin/public'; import { useLogMlJobIdFormatsShimContext } from '../../../pages/logs/shared/use_log_ml_job_id_formats_shim'; -import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; -import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../common/http_api/latest'; import { DatasetFilter } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts index 6e1927e3f7ea6..5cf26d75e9f00 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_configuration.ts @@ -7,7 +7,7 @@ import { useMemo } from 'react'; import equal from 'fast-deep-equal'; -import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { JobType } from '../../../../common/http_api/latest'; import { JobSummary } from './api/ml_get_jobs_summary_api'; import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx index 818516b8cc30d..b75f913170326 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_definition.tsx @@ -6,7 +6,7 @@ */ import { useCallback, useMemo, useState } from 'react'; -import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../common/http_api/latest'; import { getJobId } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx index 724ec3b19e31e..b6ce4085286e6 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_status.tsx @@ -7,7 +7,7 @@ import { useReducer } from 'react'; -import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../common/http_api/latest'; import { JobStatus, getDatafeedId, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts index 30ca6f730bfb0..2e0a6b742ce11 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { IdFormat, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, JobType } from '../../../../common/http_api/latest'; import { ValidateLogEntryDatasetsResponsePayload, ValidationIndicesResponsePayload, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts index f96e336080f10..14251d5f4dbf9 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.ts @@ -8,7 +8,7 @@ import { isEqual } from 'lodash'; import { useCallback, useEffect, useMemo, useState } from 'react'; import usePrevious from 'react-use/lib/usePrevious'; -import { JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { JobType } from '../../../../common/http_api/latest'; import { combineDatasetFilters, DatasetFilter, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts index ecfba10da19e6..0ed6468960ec6 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/module_descriptor.ts @@ -8,7 +8,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { i18n } from '@kbn/i18n'; import type { HttpHandler } from '@kbn/core/public'; -import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../../common/http_api/latest'; import { bucketSpan, categoriesMessageField, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx index a1c5192501970..a8c371fa9439e 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_module.tsx @@ -8,7 +8,7 @@ import createContainer from 'constate'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { useMemo } from 'react'; -import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../../common/http_api/latest'; import { useLogAnalysisModule } from '../../log_analysis_module'; import { useLogAnalysisModuleConfiguration } from '../../log_analysis_module_configuration'; import { useLogAnalysisModuleDefinition } from '../../log_analysis_module_definition'; diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts index 7101e8ec99465..61c941564f63c 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/module_descriptor.ts @@ -8,7 +8,7 @@ import { i18n } from '@kbn/i18n'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { HttpHandler } from '@kbn/core/public'; -import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../../common/http_api/latest'; import { bucketSpan, DatasetFilter, diff --git a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx index ed5a977e38cbb..a00d9f0017c31 100644 --- a/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx +++ b/x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_rate/use_log_entry_rate_module.tsx @@ -8,7 +8,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import createContainer from 'constate'; import { useMemo } from 'react'; -import { IdFormat } from '../../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../../common/http_api/latest'; import { ModuleSourceConfiguration } from '../../log_analysis_module_types'; import { useLogAnalysisModule } from '../../log_analysis_module'; import { useLogAnalysisModuleConfiguration } from '../../log_analysis_module_configuration'; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx index c2ccd01b1632c..e1db34f31a5e0 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_results_content.tsx @@ -34,7 +34,7 @@ import { StringTimeRange, useLogEntryCategoriesResultsUrlState, } from './use_log_entry_categories_results_url_state'; -import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../common/http_api/latest'; const JOB_STATUS_POLLING_INTERVAL = 30000; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts index f37d1bfd68ebf..4a194eb1fb076 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_datasets.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../common/http_api/latest'; import { getLogEntryCategoryDatasetsRequestPayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts index 18fdcfe75631a..dece712414ce6 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_log_entry_category_examples.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../common/http_api/latest'; import { getLogEntryCategoryExamplesRequestPayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts index 8cdf92a5e6e6c..2b2808bf46464 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/service_calls/get_top_log_entry_categories.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../common/http_api/latest'; import { getLogEntryCategoriesRequestPayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts index 979b8dcd6cdb0..030c0298d011e 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_categories_results.ts @@ -8,7 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../common/http_api/latest'; import { GetLogEntryCategoriesSuccessResponsePayload, GetLogEntryCategoryDatasetsSuccessResponsePayload, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx index 801c5da57ee0e..bef6675740865 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx @@ -8,7 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../common/http_api/latest'; import { LogEntryCategoryExample } from '../../../../common/http_api'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx index d258e707094b2..59535014037cd 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx @@ -15,7 +15,7 @@ import { useKibana } from '@kbn/kibana-react-plugin/public'; import { MLJobsAwaitingNodeWarning } from '@kbn/ml-plugin/public'; import { useTrackPageview } from '@kbn/observability-shared-plugin/public'; import { useLogViewContext, LogEntryFlyout } from '@kbn/logs-shared-plugin/public'; -import { IdFormatByJobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormatByJobType } from '../../../../common/http_api/latest'; import { isJobStatusWithResults, logEntryCategoriesJobType, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts index 19ffc3691540f..5f6ad4deda08a 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormatByJobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormatByJobType } from '../../../../../common/http_api/latest'; import { getLogEntryAnomaliesRequestPayloadRT, getLogEntryAnomaliesSuccessReponsePayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts index 1795fc50111f7..38ed7144140a6 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_anomalies_datasets.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormatByJobType } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormatByJobType } from '../../../../../common/http_api/latest'; import { decodeOrThrow } from '../../../../../common/runtime_types'; import { getLogEntryAnomaliesDatasetsRequestPayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts index 36c0239ddeabb..a4abfbd15ba5c 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/service_calls/get_log_entry_examples.ts @@ -7,7 +7,7 @@ import type { HttpHandler } from '@kbn/core/public'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../../common/http_api/latest'; import { getLogEntryExamplesRequestPayloadRT, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts index 420ee814996dd..94846e6b2a0e1 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts @@ -8,7 +8,7 @@ import { useMemo, useState, useCallback, useEffect, useReducer } from 'react'; import useMount from 'react-use/lib/useMount'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormatByJobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormatByJobType } from '../../../../common/http_api/latest'; import { useTrackedPromise, CanceledPromiseError } from '../../../utils/use_tracked_promise'; import { callGetLogEntryAnomaliesAPI } from './service_calls/get_log_entry_anomalies'; import { callGetLogEntryAnomaliesDatasetsAPI } from './service_calls/get_log_entry_anomalies_datasets'; diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts index ca247664cb807..86992c920a6b1 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts @@ -8,7 +8,7 @@ import { useMemo, useState } from 'react'; import { PersistedLogViewReference } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../../common/http_api/latest'; import { LogEntryExample } from '../../../../common/log_analysis'; import { useKibanaContextForPlugin } from '../../../hooks/use_kibana'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; diff --git a/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts b/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts index e13ab2408f6b5..7099e9fe9a762 100644 --- a/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts +++ b/x-pack/plugins/infra/public/pages/logs/shared/call_get_log_analysis_id_formats.ts @@ -11,7 +11,7 @@ import { getLogAnalysisIdFormatsRequestPayloadRT, getLogAnalysisIdFormatsSuccessResponsePayloadRT, LOG_ANALYSIS_GET_ID_FORMATS, -} from '../../../../common/http_api/log_analysis/v1/id_formats'; +} from '../../../../common/http_api/latest'; interface RequestArgs { spaceId: string; diff --git a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx index 1a7e1bdc27ebc..5809d5427f1ca 100644 --- a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx +++ b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx @@ -8,7 +8,7 @@ import createContainer from 'constate'; import { useState, useEffect, useMemo, useCallback } from 'react'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; -import { IdFormatByJobType, JobType } from '../../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormatByJobType, JobType } from '../../../../common/http_api/latest'; import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts index 23fc34dd5f7fc..41152d17572a8 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { PersistedLogViewReference, ResolvedLogView } from '@kbn/logs-shared-plugin/common'; -import { IdFormat, IdFormatByJobType } from '../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, IdFormatByJobType } from '../../../common/http_api/latest'; import { AnomaliesSort, getJobId, diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts index c92443b07c401..b9e908bf49eee 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts @@ -12,7 +12,7 @@ import { PersistedLogViewReference, ResolvedLogView, } from '@kbn/logs-shared-plugin/common'; -import { IdFormat } from '../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../common/http_api/latest'; import { CategoriesSort, compareDatasetsByMaximumAnomalyScore, diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts index a7df06e25bd98..5231fb5dc14ec 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IdFormat } from '../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '../../../common/http_api/latest'; import { decodeOrThrow } from '../../../common/runtime_types'; import { logRateModelPlotResponseRT, diff --git a/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts b/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts index c4a1d61e5d694..48f3b3e2fdc55 100644 --- a/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts +++ b/x-pack/plugins/infra/server/lib/log_analysis/resolve_id_formats.ts @@ -6,11 +6,7 @@ */ import { MlAnomalyDetectors } from '@kbn/ml-plugin/server'; -import { - IdFormat, - IdFormatByJobType, - JobType, -} from '../../../common/http_api/log_analysis/v1/id_formats'; +import { IdFormat, IdFormatByJobType, JobType } from '../../../common/http_api/latest'; import { getJobId, logEntryCategoriesJobType, diff --git a/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts b/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts index 7802b630af98d..800d43df448d5 100644 --- a/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts +++ b/x-pack/plugins/infra/server/routes/log_analysis/id_formats.ts @@ -7,7 +7,11 @@ import Boom from '@hapi/boom'; import { createValidationFunction } from '@kbn/logs-shared-plugin/common/runtime_types'; -import * as logAnalysisIdFormatsV1 from '../../../common/http_api/log_analysis/v1/id_formats'; +import { + LOG_ANALYSIS_GET_ID_FORMATS, + getLogAnalysisIdFormatsRequestPayloadRT, + getLogAnalysisIdFormatsSuccessResponsePayloadRT, +} from '../../../common/http_api/latest'; import { InfraBackendLibs } from '../../lib/infra_types'; import { isMlPrivilegesError } from '../../lib/log_analysis'; import { resolveIdFormats } from '../../lib/log_analysis/resolve_id_formats'; @@ -18,16 +22,14 @@ export const initGetLogAnalysisIdFormatsRoute = ({ framework }: InfraBackendLibs .registerVersionedRoute({ access: 'internal', method: 'post', - path: logAnalysisIdFormatsV1.LOG_ANALYSIS_GET_ID_FORMATS, + path: LOG_ANALYSIS_GET_ID_FORMATS, }) .addVersion( { version: '1', validate: { request: { - body: createValidationFunction( - logAnalysisIdFormatsV1.getLogAnalysisIdFormatsRequestPayloadRT - ), + body: createValidationFunction(getLogAnalysisIdFormatsRequestPayloadRT), }, }, }, @@ -43,7 +45,7 @@ export const initGetLogAnalysisIdFormatsRoute = ({ framework }: InfraBackendLibs const idFormatByJobType = await resolveIdFormats(logViewId, spaceId, mlAnomalyDetectors); return response.ok({ - body: logAnalysisIdFormatsV1.getLogAnalysisIdFormatsSuccessResponsePayloadRT.encode({ + body: getLogAnalysisIdFormatsSuccessResponsePayloadRT.encode({ data: idFormatByJobType, }), }); diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts b/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts index 9a90dcd07eb50..fb89001425d94 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts +++ b/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { IdFormat } from '@kbn/infra-plugin/common/http_api/log_analysis/v1/id_formats'; +import { IdFormat } from '@kbn/infra-plugin/common/http_api/latest'; const rateHashedPattern = /logs-[0-9a-fA-F]{32,}-log-entry-rate/; const rateLegacyPattern = /kibana-logs-ui-.*-.*-log-entry-rate/; From 0b4e168a1913d7b0ab9d674d68e68577351d441b Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Mon, 13 Nov 2023 15:46:57 +0100 Subject: [PATCH 08/11] Remove unnecessary useMemo calls, use previous value in state update and add guard --- .../shared/use_log_ml_job_id_formats_shim.tsx | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx index 5809d5427f1ca..47eb778185762 100644 --- a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx +++ b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx @@ -53,25 +53,21 @@ const useLogMlJobIdFormatsShim = () => { getLogAnalysisIdFormats(); }, [getLogAnalysisIdFormats]); - const isLoadingLogAnalysisIdFormats = useMemo( - () => getLogAnalysisIdFormatsRequest.state === 'pending', - [getLogAnalysisIdFormatsRequest.state] - ); + const isLoadingLogAnalysisIdFormats = getLogAnalysisIdFormatsRequest.state === 'pending'; + const hasFailedLoadingLogAnalysisIdFormats = getLogAnalysisIdFormatsRequest.state === 'rejected'; - const hasFailedLoadingLogAnalysisIdFormats = useMemo( - () => getLogAnalysisIdFormatsRequest.state === 'rejected', - [getLogAnalysisIdFormatsRequest.state] - ); + const migrateIdFormat = useCallback((jobType: JobType) => { + setIdFormats((previousValue) => { + if (!previousValue) { + return null; + } - const migrateIdFormat = useCallback( - (jobType: JobType) => { - setIdFormats({ - ...idFormats!, + return { + ...previousValue, [jobType]: 'hashed', - }); - }, - [idFormats] - ); + }; + }); + }, []); return { idFormats, From a6baa5a108b9821e5c0b6e52b8b24cd97d915df6 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Mon, 13 Nov 2023 16:17:25 +0100 Subject: [PATCH 09/11] Shuffle around null checks --- .../pages/logs/log_entry_categories/page_providers.tsx | 4 ++-- .../sections/top_categories/category_details_row.tsx | 2 +- .../use_log_entry_category_examples.tsx | 6 +++++- .../public/pages/logs/log_entry_rate/page_content.tsx | 2 +- .../pages/logs/log_entry_rate/page_providers.tsx | 6 +++--- .../pages/logs/log_entry_rate/page_results_content.tsx | 2 +- .../log_entry_rate/sections/anomalies/expanded_row.tsx | 2 +- .../log_entry_rate/use_log_entry_anomalies_results.ts | 10 +++++++++- .../logs/log_entry_rate/use_log_entry_examples.ts | 6 +++++- 9 files changed, 28 insertions(+), 12 deletions(-) diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx index 74b041e466f55..89bcd98d0958f 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx @@ -39,7 +39,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child return ; } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) { return ; - } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats) { + } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats || !idFormats) { return ; } else if (resolvedLogView != null) { if (logViewReference.type === 'log-view-inline') { @@ -50,7 +50,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} - idFormat={idFormats![logEntryCategoriesJobType]} + idFormat={idFormats[logEntryCategoriesJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_details_row.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_details_row.tsx index 133f11eef5d1b..c6fccd3d4ff82 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_details_row.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/sections/top_categories/category_details_row.tsx @@ -33,7 +33,7 @@ export const CategoryDetailsRow: React.FunctionComponent<{ endTime: timeRange.endTime, exampleCount, logViewReference, - idFormat: idFormats![logEntryCategoriesJobType], + idFormat: idFormats?.[logEntryCategoriesJobType], startTime: timeRange.startTime, }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx index bef6675740865..e50664b957d74 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_categories/use_log_entry_category_examples.tsx @@ -26,7 +26,7 @@ export const useLogEntryCategoryExamples = ({ endTime: number; exampleCount: number; logViewReference: PersistedLogViewReference; - idFormat: IdFormat; + idFormat?: IdFormat; startTime: number; }) => { const { services } = useKibanaContextForPlugin(); @@ -39,6 +39,10 @@ export const useLogEntryCategoryExamples = ({ { cancelPreviousOn: 'creation', createPromise: async () => { + if (!idFormat) { + throw new Error('idFormat is undefined'); + } + return await callGetLogEntryCategoryExamplesAPI( { logViewReference, diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx index e047408aec48c..e4dc0694c3f75 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_content.tsx @@ -137,7 +137,7 @@ export const LogEntryRatePageContent = memo(() => { ) { return ( <> - + ); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx index e7ba080b6af02..273874f83ae3e 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_providers.tsx @@ -41,7 +41,7 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) return null; } else if (!isPersistedLogView) { return ; - } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats) { + } else if (isLoading || isUninitialized || isLoadingLogAnalysisIdFormats || !idFormats) { return ; } else if (hasFailedLoading || hasFailedLoadingLogAnalysisIdFormats) { return ; @@ -55,7 +55,7 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} - idFormat={idFormats![logEntryRateJobType]} + idFormat={idFormats[logEntryRateJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > @@ -63,7 +63,7 @@ export const LogEntryRatePageProviders: React.FunctionComponent = ({ children }) indexPattern={resolvedLogView.indices} logViewId={logViewReference.logViewId} spaceId={space.id} - idFormat={idFormats![logEntryCategoriesJobType]} + idFormat={idFormats[logEntryCategoriesJobType]} timestampField={resolvedLogView.timestampField} runtimeMappings={resolvedLogView.runtimeMappings} > diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx index 59535014037cd..21eb5aacb2cec 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/page_results_content.tsx @@ -50,7 +50,7 @@ export const PAGINATION_DEFAULTS = { export const LogEntryRateResultsContent: React.FunctionComponent<{ pageTitle: string; - idFormats: IdFormatByJobType; + idFormats: IdFormatByJobType | null; }> = ({ pageTitle, idFormats }) => { useTrackPageview({ app: 'infra_logs', path: 'log_entry_rate_results' }); useTrackPageview({ app: 'infra_logs', path: 'log_entry_rate_results', delay: 15000 }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx index 83e8184d34c93..8b4ea2edda9ac 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/sections/anomalies/expanded_row.tsx @@ -50,7 +50,7 @@ export const AnomaliesTableExpandedRow: React.FunctionComponent<{ endTime: anomaly.startTime + anomaly.duration, exampleCount: EXAMPLE_COUNT, logViewReference, - idFormat: idFormats![logEntryRateJobType], + idFormat: idFormats?.[logEntryRateJobType], startTime: anomaly.startTime, categoryId: isCategoryAnomaly(anomaly) ? anomaly.categoryId : undefined, }); diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts index 94846e6b2a0e1..598f57751dae6 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_anomalies_results.ts @@ -149,7 +149,7 @@ export const useLogEntryAnomaliesResults = ({ endTime: number; startTime: number; logViewReference: PersistedLogViewReference; - idFormats: IdFormatByJobType; + idFormats: IdFormatByJobType | null; defaultSortOptions: AnomaliesSort; defaultPaginationOptions: Pick; onGetLogEntryAnomaliesDatasetsError?: (error: Error) => void; @@ -178,6 +178,10 @@ export const useLogEntryAnomaliesResults = ({ { cancelPreviousOn: 'creation', createPromise: async () => { + if (!idFormats) { + throw new Error('idFormats is undefined'); + } + const { timeRange: { start: queryStartTime, end: queryEndTime }, sortOptions, @@ -299,6 +303,10 @@ export const useLogEntryAnomaliesResults = ({ { cancelPreviousOn: 'creation', createPromise: async () => { + if (!idFormats) { + throw new Error('idFormats is undefined'); + } + return await callGetLogEntryAnomaliesDatasetsAPI( { logViewReference, idFormats, startTime, endTime }, services.http.fetch diff --git a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts index 86992c920a6b1..58e7b15842a17 100644 --- a/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts +++ b/x-pack/plugins/infra/public/pages/logs/log_entry_rate/use_log_entry_examples.ts @@ -27,7 +27,7 @@ export const useLogEntryExamples = ({ endTime: number; exampleCount: number; logViewReference: PersistedLogViewReference; - idFormat: IdFormat; + idFormat?: IdFormat; startTime: number; categoryId?: string; }) => { @@ -38,6 +38,10 @@ export const useLogEntryExamples = ({ { cancelPreviousOn: 'creation', createPromise: async () => { + if (!idFormat) { + throw new Error('idFormat is undefined'); + } + return await callGetLogEntryExamplesAPI( { logViewReference, From 1e57ce5e0a7121c75a2d736942055e806089fd47 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:53:23 +0000 Subject: [PATCH 10/11] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx index 47eb778185762..ffa63908b8b57 100644 --- a/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx +++ b/x-pack/plugins/infra/public/pages/logs/shared/use_log_ml_job_id_formats_shim.tsx @@ -6,7 +6,7 @@ */ import createContainer from 'constate'; -import { useState, useEffect, useMemo, useCallback } from 'react'; +import { useState, useEffect, useCallback } from 'react'; import { useLogViewContext } from '@kbn/logs-shared-plugin/public'; import { IdFormatByJobType, JobType } from '../../../../common/http_api/latest'; import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space'; From 0d2fb437bfc68217bc0ab88173c82d3e2a894881 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 15 Nov 2023 12:07:12 +0100 Subject: [PATCH 11/11] Move tests to new location --- x-pack/test/functional/apps/infra/index.ts | 2 +- .../infra/{ => logs}/ml_job_id_formats/assert_id_formats.ts | 0 .../apps/infra/{ => logs}/ml_job_id_formats/ml_job_configs.ts | 0 .../apps/infra/{ => logs}/ml_job_id_formats/ml_job_helper.ts | 2 +- .../infra/{ => logs}/ml_job_id_formats/request_tracker.ts | 4 ++-- .../apps/infra/{ => logs}/ml_job_id_formats/tests.ts | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename x-pack/test/functional/apps/infra/{ => logs}/ml_job_id_formats/assert_id_formats.ts (100%) rename x-pack/test/functional/apps/infra/{ => logs}/ml_job_id_formats/ml_job_configs.ts (100%) rename x-pack/test/functional/apps/infra/{ => logs}/ml_job_id_formats/ml_job_helper.ts (94%) rename x-pack/test/functional/apps/infra/{ => logs}/ml_job_id_formats/request_tracker.ts (94%) rename x-pack/test/functional/apps/infra/{ => logs}/ml_job_id_formats/tests.ts (99%) diff --git a/x-pack/test/functional/apps/infra/index.ts b/x-pack/test/functional/apps/infra/index.ts index 98e4387e31e43..5ad5c004c30c9 100644 --- a/x-pack/test/functional/apps/infra/index.ts +++ b/x-pack/test/functional/apps/infra/index.ts @@ -29,7 +29,7 @@ export default ({ loadTestFile }: FtrProviderContext) => { loadTestFile(require.resolve('./logs/log_stream_date_nano')); loadTestFile(require.resolve('./logs/link_to')); loadTestFile(require.resolve('./logs/log_stream')); - loadTestFile(require.resolve('./ml_job_id_formats/tests')); + loadTestFile(require.resolve('./logs/ml_job_id_formats/tests')); }); }); }; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/assert_id_formats.ts similarity index 100% rename from x-pack/test/functional/apps/infra/ml_job_id_formats/assert_id_formats.ts rename to x-pack/test/functional/apps/infra/logs/ml_job_id_formats/assert_id_formats.ts diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/ml_job_configs.ts similarity index 100% rename from x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_configs.ts rename to x-pack/test/functional/apps/infra/logs/ml_job_id_formats/ml_job_configs.ts diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/ml_job_helper.ts similarity index 94% rename from x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts rename to x-pack/test/functional/apps/infra/logs/ml_job_id_formats/ml_job_helper.ts index 3f4f85d47f339..7c6dcc9b5575f 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/ml_job_helper.ts +++ b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/ml_job_helper.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { MlApi } from '../../../services/ml/api'; +import { MlApi } from '../../../../services/ml/api'; import { MlJob } from './ml_job_configs'; export function createMlJobHelper(ml: MlApi) { diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/request_tracker.ts similarity index 94% rename from x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts rename to x-pack/test/functional/apps/infra/logs/ml_job_id_formats/request_tracker.ts index ae7905e342a6c..fe7fb5a49b15d 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/request_tracker.ts +++ b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/request_tracker.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { Browser } from '../../../../../../test/functional/services/common/browser'; -import { CommonPageObject } from '../../../../../../test/functional/page_objects/common_page'; +import { Browser } from '../../../../../../../test/functional/services/common/browser'; +import { CommonPageObject } from '../../../../../../../test/functional/page_objects/common_page'; type PerformanceResourceTimingWithHttpStatus = PerformanceResourceTiming & { responseStatus: number; diff --git a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts similarity index 99% rename from x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts rename to x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts index 4625eee74383b..4e0e12a386dc9 100644 --- a/x-pack/test/functional/apps/infra/ml_job_id_formats/tests.ts +++ b/x-pack/test/functional/apps/infra/logs/ml_job_id_formats/tests.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { FtrProviderContext } from '../../../ftr_provider_context'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; import { assertIdFormats } from './assert_id_formats'; import { createMlJobHelper, MlJobHelper } from './ml_job_helper'; import { createRequestTracker } from './request_tracker';