diff --git a/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts index 82cfb0f83ed69..ebdc2251891eb 100644 --- a/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts +++ b/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts @@ -6,8 +6,13 @@ import { JobType } from './log_analysis'; +export const bucketSpan = 900000; + export const getJobIdPrefix = (spaceId: string, sourceId: string) => `kibana-logs-ui-${spaceId}-${sourceId}-`; export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) => `${getJobIdPrefix(spaceId, sourceId)}${jobType}`; + +export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) => + `datafeed-${getJobId(spaceId, sourceId, jobType)}`; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts new file mode 100644 index 0000000000000..5185a6f250ace --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts @@ -0,0 +1,130 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as rt from 'io-ts'; +import { kfetch } from 'ui/kfetch'; + +import { getJobIdPrefix } from '../../../../../common/log_analysis'; +import { throwErrors, createPlainError } from '../../../../../common/runtime_types'; + +const MODULE_ID = 'logs_ui_analysis'; + +// This is needed due to: https://github.com/elastic/kibana/issues/43671 +const removeSampleDataIndex = (indexPattern: string) => { + const SAMPLE_DATA_INDEX = 'kibana_sample_data_logs*'; + const indices = indexPattern.split(','); + const sampleDataIndex = indices.findIndex((index: string) => { + return index === SAMPLE_DATA_INDEX; + }); + if (sampleDataIndex > -1) { + indices.splice(sampleDataIndex, 1); + return indices.join(','); + } else { + return indexPattern; + } +}; + +export const callSetupMlModuleAPI = async ( + start: number | undefined, + end: number | undefined, + spaceId: string, + sourceId: string, + indexPattern: string, + timeField: string, + bucketSpan: number +) => { + const response = await kfetch({ + method: 'POST', + pathname: `/api/ml/modules/setup/${MODULE_ID}`, + body: JSON.stringify( + setupMlModuleRequestPayloadRT.encode({ + start, + end, + indexPatternName: removeSampleDataIndex(indexPattern), + prefix: getJobIdPrefix(spaceId, sourceId), + startDatafeed: true, + jobOverrides: [ + { + job_id: 'log-entry-rate', + analysis_config: { + bucket_span: `${bucketSpan}ms`, + }, + data_description: { + time_field: timeField, + }, + }, + ], + datafeedOverrides: [ + { + job_id: 'log-entry-rate', + aggregations: { + buckets: { + date_histogram: { + field: timeField, + fixed_interval: `${bucketSpan}ms`, + }, + aggregations: { + [timeField]: { + max: { + field: `${timeField}`, + }, + }, + doc_count_per_minute: { + bucket_script: { + script: { + params: { + bucket_span_in_ms: bucketSpan, + }, + }, + }, + }, + }, + }, + }, + }, + ], + }) + ), + }); + + return setupMlModuleResponsePayloadRT.decode(response).getOrElseL(throwErrors(createPlainError)); +}; + +const setupMlModuleTimeParamsRT = rt.partial({ + start: rt.number, + end: rt.number, +}); + +const setupMlModuleRequestParamsRT = rt.type({ + indexPatternName: rt.string, + prefix: rt.string, + startDatafeed: rt.boolean, + jobOverrides: rt.array(rt.object), + datafeedOverrides: rt.array(rt.object), +}); + +const setupMlModuleRequestPayloadRT = rt.intersection([ + setupMlModuleTimeParamsRT, + setupMlModuleRequestParamsRT, +]); + +const setupMlModuleResponsePayloadRT = rt.type({ + datafeeds: rt.array( + rt.type({ + id: rt.string, + started: rt.boolean, + success: rt.boolean, + }) + ), + jobs: rt.array( + rt.type({ + id: rt.string, + success: rt.boolean, + }) + ), +}); + +export type SetupMlModuleResponsePayload = rt.TypeOf; diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx index 391a4b190edd2..458fc5bb8635a 100644 --- a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx +++ b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx @@ -6,22 +6,31 @@ import createContainer from 'constate-latest'; import { useMemo, useEffect, useState } from 'react'; -import { values } from 'lodash'; -import { getJobId } from '../../../../common/log_analysis'; +import { bucketSpan, getJobId } from '../../../../common/log_analysis'; import { useTrackedPromise } from '../../../utils/use_tracked_promise'; +import { callSetupMlModuleAPI, SetupMlModuleResponsePayload } from './api/ml_setup_module_api'; import { callJobsSummaryAPI } from './api/ml_get_jobs_summary_api'; -type JobStatus = 'unknown' | 'closed' | 'closing' | 'failed' | 'opened' | 'opening' | 'deleted'; -// type DatafeedStatus = 'unknown' | 'started' | 'starting' | 'stopped' | 'stopping' | 'deleted'; +// combines and abstracts job and datafeed status +type JobStatus = + | 'unknown' + | 'missing' + | 'inconsistent' + | 'created' + | 'started' + | 'opening' + | 'opened'; export const useLogAnalysisJobs = ({ indexPattern, sourceId, spaceId, + timeField, }: { indexPattern: string; sourceId: string; spaceId: string; + timeField: string; }) => { const [jobStatus, setJobStatus] = useState<{ logEntryRate: JobStatus; @@ -29,25 +38,38 @@ export const useLogAnalysisJobs = ({ logEntryRate: 'unknown', }); - // const [setupMlModuleRequest, setupMlModule] = useTrackedPromise( - // { - // cancelPreviousOn: 'resolution', - // createPromise: async () => { - // kfetch({ - // method: 'POST', - // pathname: '/api/ml/modules/setup', - // body: JSON.stringify( - // setupMlModuleRequestPayloadRT.encode({ - // indexPatternName: indexPattern, - // prefix: getJobIdPrefix(spaceId, sourceId), - // startDatafeed: true, - // }) - // ), - // }); - // }, - // }, - // [indexPattern, spaceId, sourceId] - // ); + const [setupMlModuleRequest, setupMlModule] = useTrackedPromise( + { + cancelPreviousOn: 'resolution', + createPromise: async (start, end) => { + return await callSetupMlModuleAPI( + start, + end, + spaceId, + sourceId, + indexPattern, + timeField, + bucketSpan + ); + }, + onResolve: ({ datafeeds, jobs }: SetupMlModuleResponsePayload) => { + const hasSuccessfullyCreatedJobs = jobs.every(job => job.success); + const hasSuccessfullyStartedDatafeeds = datafeeds.every( + datafeed => datafeed.success && datafeed.started + ); + + setJobStatus(currentJobStatus => ({ + ...currentJobStatus, + logEntryRate: hasSuccessfullyCreatedJobs + ? hasSuccessfullyStartedDatafeeds + ? 'started' + : 'created' + : 'inconsistent', + })); + }, + }, + [indexPattern, spaceId, sourceId] + ); const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise( { @@ -77,9 +99,10 @@ export const useLogAnalysisJobs = ({ }, []); const isSetupRequired = useMemo(() => { - const jobStates = values(jobStatus); + const jobStates = Object.values(jobStatus); return ( - jobStates.filter(state => state === 'opened' || state === 'opening').length < jobStates.length + jobStates.filter(state => ['opened', 'opening', 'created', 'started'].includes(state)) + .length < jobStates.length ); }, [jobStatus]); @@ -87,10 +110,23 @@ export const useLogAnalysisJobs = ({ fetchJobStatusRequest.state, ]); + const isSettingUpMlModule = useMemo(() => setupMlModuleRequest.state === 'pending', [ + setupMlModuleRequest.state, + ]); + + const didSetupFail = useMemo( + () => !isSettingUpMlModule && setupMlModuleRequest.state !== 'uninitialized' && isSetupRequired, + [setupMlModuleRequest.state, jobStatus] + ); + return { jobStatus, isSetupRequired, isLoadingSetupStatus, + setupMlModule, + setupMlModuleRequest, + isSettingUpMlModule, + didSetupFail, }; }; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/analysis_setup_timerange_form.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/analysis_setup_timerange_form.tsx new file mode 100644 index 0000000000000..120670db83e8a --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/analysis_setup_timerange_form.tsx @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useMemo, useState } from 'react'; +import moment, { Moment } from 'moment'; + +import { i18n } from '@kbn/i18n'; +import { + EuiForm, + EuiDescribedFormGroup, + EuiFormRow, + EuiDatePicker, + EuiFlexGroup, + EuiFormControlLayout, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { CreateMLJobsButton } from './create_ml_jobs_button'; + +const startTimeLabel = i18n.translate('xpack.infra.analysisSetup.startTimeLabel', { + defaultMessage: 'Start time', +}); +const endTimeLabel = i18n.translate('xpack.infra.analysisSetup.endTimeLabel', { + defaultMessage: 'End time', +}); +const startTimeDefaultDescription = i18n.translate( + 'xpack.infra.analysisSetup.startTimeDefaultDescription', + { + defaultMessage: 'Start of log indices', + } +); +const endTimeDefaultDescription = i18n.translate( + 'xpack.infra.analysisSetup.endTimeDefaultDescription', + { + defaultMessage: 'Indefinitely', + } +); + +function selectedDateToParam(selectedDate: Moment | null) { + if (selectedDate) { + return selectedDate.valueOf(); // To ms unix timestamp + } + return undefined; +} + +export const AnalysisSetupTimerangeForm: React.FunctionComponent<{ + isSettingUp: boolean; + setupMlModule: (startTime: number | undefined, endTime: number | undefined) => Promise; +}> = ({ isSettingUp, setupMlModule }) => { + const [startTime, setStartTime] = useState(null); + const [endTime, setEndTime] = useState(null); + + const now = useMemo(() => moment(), []); + const selectedEndTimeIsToday = !endTime || endTime.isSame(now, 'day'); + + const onClickCreateJob = () => + setupMlModule(selectedDateToParam(startTime), selectedDateToParam(endTime)); + + return ( + + + } + description={ + + } + > + + + setStartTime(null) } : undefined} + > + + + + + + + setEndTime(null) } : undefined}> + + + + + + + + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/create_ml_jobs_button.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/create_ml_jobs_button.tsx new file mode 100644 index 0000000000000..24caec70ed841 --- /dev/null +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/create_ml_jobs_button.tsx @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { EuiButton } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; + +export const CreateMLJobsButton: React.FunctionComponent<{ + isLoading: boolean; + onClick: () => void; +}> = ({ isLoading, onClick }) => { + return ( + + + + ); +}; diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx index ef4a1ffe6735c..b501b8896e61e 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page.tsx @@ -18,10 +18,17 @@ import { Source } from '../../../containers/source'; export const AnalysisPage = () => { const { sourceId, source } = useContext(Source.Context); const spaceId = chrome.getInjected('activeSpace').space.id; - const { isSetupRequired, isLoadingSetupStatus } = useLogAnalysisJobs({ + const { + isSetupRequired, + isLoadingSetupStatus, + setupMlModule, + isSettingUpMlModule, + didSetupFail, + } = useLogAnalysisJobs({ indexPattern: source ? source.configuration.logAlias : '', sourceId, spaceId, + timeField: source ? source.configuration.fields.timestamp : '', }); return ( @@ -34,7 +41,11 @@ export const AnalysisPage = () => { })} /> ) : isSetupRequired ? ( - + ) : ( )} diff --git a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx index a4e1402e9ce60..248d4d7bbb3b8 100644 --- a/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx +++ b/x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx @@ -4,13 +4,136 @@ * you may not use this file except in compliance with the Elastic License. */ -import React from 'react'; - +import React, { useState } from 'react'; +import { + EuiButtonEmpty, + EuiPage, + EuiPageBody, + EuiPageContent, + EuiPageContentHeader, + EuiPageContentHeaderSection, + EuiPageContentBody, + EuiText, + EuiTitle, + EuiSpacer, + EuiCallOut, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import euiStyled from '../../../../../../common/eui_styled_components'; import { useTrackPageview } from '../../../hooks/use_track_metric'; -export const AnalysisSetupContent = () => { +import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form'; +import { CreateMLJobsButton } from './create_ml_jobs_button'; + +interface AnalysisSetupContentProps { + setupMlModule: (startTime?: number | undefined, endTime?: number | undefined) => Promise; + isSettingUp: boolean; + didSetupFail: boolean; +} + +const errorTitle = i18n.translate('xpack.infra.analysisSetup.errorTitle', { + defaultMessage: 'Sorry, there was an error setting up Machine Learning', +}); + +export const AnalysisSetupContent: React.FunctionComponent = ({ + setupMlModule, + isSettingUp, + didSetupFail, +}) => { useTrackPageview({ app: 'infra_logs', path: 'analysis_setup' }); useTrackPageview({ app: 'infra_logs', path: 'analysis_setup', delay: 15000 }); - return
Setup
; + const [showTimeRangeForm, setShowTimeRangeForm] = useState(false); + + return ( + + + + + + +

+ +

+
+
+
+ + + + + {showTimeRangeForm ? ( + <> + + + + ) : ( + <> + + + {' '} + setShowTimeRangeForm(true)}> + + + + + setupMlModule()} /> + + )} + {didSetupFail && ( + + + + + + )} + +
+
+
+ ); }; + +// !important due to https://github.com/elastic/eui/issues/2232 +const AnalysisPageContent = euiStyled(EuiPageContent)` + max-width: 518px !important; +`; + +const AnalysisSetupPage = euiStyled(EuiPage)` + height: 100%; +`; + +const ByDefaultText = euiStyled(EuiText).attrs({ size: 's' })` + & .euiButtonEmpty { + font-size: inherit; + line-height: inherit; + height: initial; + } + + & .euiButtonEmpty__content { + padding: 0; + } +`;