[Logs UI] Generalize ML module management (#50662)

This abstracts the specific job details out of the ML module management hooks to enable re-use with the upcoming categorization module. closes #50322
elastic · Dec 10, 2019 · cf28280 · cf28280
1 parent 3c57f71
commit cf28280
Show file tree

Hide file tree

Showing 61 changed files with 910 additions and 704 deletions.
diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/results/log_entry_rate.ts
@@ -29,7 +29,7 @@ export type GetLogEntryRateRequestPayload = rt.TypeOf<typeof getLogEntryRateRequ
  * response
  */
 
-export const logEntryRateAnomaly = rt.type({
+export const logEntryRateAnomalyRT = rt.type({
   actualLogEntryRate: rt.number,
   anomalyScore: rt.number,
   duration: rt.number,
@@ -39,22 +39,26 @@ export const logEntryRateAnomaly = rt.type({
 
 export const logEntryRatePartitionRT = rt.type({
   analysisBucketCount: rt.number,
-  anomalies: rt.array(logEntryRateAnomaly),
+  anomalies: rt.array(logEntryRateAnomalyRT),
   averageActualLogEntryRate: rt.number,
   maximumAnomalyScore: rt.number,
   numberOfLogEntries: rt.number,
   partitionId: rt.string,
 });
 
-export const logEntryRateHistogramBucket = rt.type({
+export type LogEntryRatePartition = rt.TypeOf<typeof logEntryRatePartitionRT>;
+
+export const logEntryRateHistogramBucketRT = rt.type({
   partitions: rt.array(logEntryRatePartitionRT),
   startTime: rt.number,
 });
 
+export type LogEntryRateHistogramBucket = rt.TypeOf<typeof logEntryRateHistogramBucketRT>;
+
 export const getLogEntryRateSuccessReponsePayloadRT = rt.type({
   data: rt.type({
     bucketDuration: rt.number,
-    histogramBuckets: rt.array(logEntryRateHistogramBucket),
+    histogramBuckets: rt.array(logEntryRateHistogramBucketRT),
     totalNumberOfLogEntries: rt.number,
   }),
 });

diff --git a/x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/index.ts b/x-pack/legacy/plugins/infra/common/http_api/log_analysis/validation/index.ts
@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export * from './indices';
+export * from './log_entry_rate_indices';
diff --git a/...tp_api/log_analysis/validation/indices.ts → ...ysis/validation/log_entry_rate_indices.ts b/...tp_api/log_analysis/validation/indices.ts → ...ysis/validation/log_entry_rate_indices.ts
@@ -6,14 +6,24 @@
 
 import * as rt from 'io-ts';
 
-export const LOG_ANALYSIS_VALIDATION_INDICES_PATH = '/api/infra/log_analysis/validation/indices';
+export const LOG_ANALYSIS_VALIDATE_INDICES_PATH =
+  '/api/infra/log_analysis/validation/log_entry_rate_indices';
 
 /**
  * Request types
  */
+export const validationIndicesFieldSpecificationRT = rt.type({
+  name: rt.string,
+  validTypes: rt.array(rt.string),
+});
+
+export type ValidationIndicesFieldSpecification = rt.TypeOf<
+  typeof validationIndicesFieldSpecificationRT
+>;
+
 export const validationIndicesRequestPayloadRT = rt.type({
   data: rt.type({
-    timestampField: rt.string,
+    fields: rt.array(validationIndicesFieldSpecificationRT),
     indices: rt.array(rt.string),
   }),
 });

diff --git a/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts b/x-pack/legacy/plugins/infra/common/log_analysis/job_parameters.ts
@@ -4,19 +4,25 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { JobType } from './log_analysis';
+import * as rt from 'io-ts';
 
 export const bucketSpan = 900000;
 
+export const partitionField = 'event.dataset';
+
 export const getJobIdPrefix = (spaceId: string, sourceId: string) =>
   `kibana-logs-ui-${spaceId}-${sourceId}-`;
 
-export const getJobId = (spaceId: string, sourceId: string, jobType: JobType) =>
+export const getJobId = (spaceId: string, sourceId: string, jobType: string) =>
   `${getJobIdPrefix(spaceId, sourceId)}${jobType}`;
 
-export const getDatafeedId = (spaceId: string, sourceId: string, jobType: JobType) =>
+export const getDatafeedId = (spaceId: string, sourceId: string, jobType: string) =>
   `datafeed-${getJobId(spaceId, sourceId, jobType)}`;
 
-export const getAllModuleJobIds = (spaceId: string, sourceId: string) => [
-  getJobId(spaceId, sourceId, 'log-entry-rate'),
-];
+export const jobSourceConfigurationRT = rt.type({
+  indexPattern: rt.string,
+  timestampField: rt.string,
+  bucketSpan: rt.number,
+});
+
+export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;
diff --git a/...nalysis/sections/analyze_in_ml_button.tsx → ...analysis_results/analyze_in_ml_button.tsx b/...nalysis/sections/analyze_in_ml_button.tsx → ...analysis_results/analyze_in_ml_button.tsx
@@ -4,14 +4,15 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React from 'react';
-import url from 'url';
 import { EuiButton } from '@elastic/eui';
 import { FormattedMessage } from '@kbn/i18n/react';
+import React from 'react';
+import { encode } from 'rison-node';
 import chrome from 'ui/chrome';
 import { QueryString } from 'ui/utils/query_string';
-import { encode } from 'rison-node';
-import { TimeRange } from '../../../../../common/http_api/shared/time_range';
+import url from 'url';
+
+import { TimeRange } from '../../../../common/http_api/shared/time_range';
 
 export const AnalyzeInMlButton: React.FunctionComponent<{
   jobId: string;

diff --git a/x-pack/legacy/plugins/infra/public/components/logging/log_analysis_results/index.ts b/x-pack/legacy/plugins/infra/public/components/logging/log_analysis_results/index.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './analyze_in_ml_button';
diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_api_types.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_api_types.ts
@@ -6,11 +6,30 @@
 
 import * as rt from 'io-ts';
 
+import { jobSourceConfigurationRT } from '../../../../../common/log_analysis';
+
 export const jobCustomSettingsRT = rt.partial({
   job_revision: rt.number,
-  logs_source_config: rt.partial({
-    indexPattern: rt.string,
-    timestampField: rt.string,
-    bucketSpan: rt.number,
+  logs_source_config: rt.partial(jobSourceConfigurationRT.props),
+});
+
+export const getMlCapabilitiesResponsePayloadRT = rt.type({
+  capabilities: rt.type({
+    canGetJobs: rt.boolean,
+    canCreateJob: rt.boolean,
+    canDeleteJob: rt.boolean,
+    canOpenJob: rt.boolean,
+    canCloseJob: rt.boolean,
+    canForecastJob: rt.boolean,
+    canGetDatafeeds: rt.boolean,
+    canStartStopDatafeed: rt.boolean,
+    canUpdateJob: rt.boolean,
+    canUpdateDatafeed: rt.boolean,
+    canPreviewDatafeed: rt.boolean,
   }),
+  isPlatinumOrTrialLicense: rt.boolean,
+  mlFeatureEnabledInSpace: rt.boolean,
+  upgradeInProgress: rt.boolean,
 });
+
+export type GetMlCapabilitiesResponsePayload = rt.TypeOf<typeof getMlCapabilitiesResponsePayloadRT>;
diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_cleanup.ts
@@ -9,17 +9,22 @@ import { pipe } from 'fp-ts/lib/pipeable';
 import { fold } from 'fp-ts/lib/Either';
 import { identity } from 'fp-ts/lib/function';
 import { kfetch } from 'ui/kfetch';
-import { getAllModuleJobIds, getDatafeedId } from '../../../../../common/log_analysis';
+
+import { getDatafeedId, getJobId } from '../../../../../common/log_analysis';
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 
-export const callDeleteJobs = async (spaceId: string, sourceId: string) => {
+export const callDeleteJobs = async <JobType extends string>(
+  spaceId: string,
+  sourceId: string,
+  jobTypes: JobType[]
+) => {
   // NOTE: Deleting the jobs via this API will delete the datafeeds at the same time
   const deleteJobsResponse = await kfetch({
     method: 'POST',
     pathname: '/api/ml/jobs/delete_jobs',
     body: JSON.stringify(
       deleteJobsRequestPayloadRT.encode({
-        jobIds: getAllModuleJobIds(spaceId, sourceId),
+        jobIds: jobTypes.map(jobType => getJobId(spaceId, sourceId, jobType)),
       })
     ),
   });
@@ -42,15 +47,24 @@ export const callGetJobDeletionTasks = async () => {
   );
 };
 
-export const callStopDatafeed = async (spaceId: string, sourceId: string) => {
+export const callStopDatafeeds = async <JobType extends string>(
+  spaceId: string,
+  sourceId: string,
+  jobTypes: JobType[]
+) => {
   // Stop datafeed due to https://github.com/elastic/kibana/issues/44652
   const stopDatafeedResponse = await kfetch({
     method: 'POST',
-    pathname: `/api/ml/datafeeds/${getDatafeedId(spaceId, sourceId, 'log-entry-rate')}/_stop`,
+    pathname: '/api/ml/jobs/stop_datafeeds',
+    body: JSON.stringify(
+      stopDatafeedsRequestPayloadRT.encode({
+        datafeedIds: jobTypes.map(jobType => getDatafeedId(spaceId, sourceId, jobType)),
+      })
+    ),
   });
 
   return pipe(
-    stopDatafeedResponsePayloadRT.decode(stopDatafeedResponse),
+    stopDatafeedsResponsePayloadRT.decode(stopDatafeedResponse),
     fold(throwErrors(createPlainError), identity)
   );
 };
@@ -68,10 +82,19 @@ export const deleteJobsResponsePayloadRT = rt.record(
   })
 );
 
+export type DeleteJobsResponsePayload = rt.TypeOf<typeof deleteJobsResponsePayloadRT>;
+
 export const getJobDeletionTasksResponsePayloadRT = rt.type({
   jobIds: rt.array(rt.string),
 });
 
-export const stopDatafeedResponsePayloadRT = rt.type({
-  stopped: rt.boolean,
+export const stopDatafeedsRequestPayloadRT = rt.type({
+  datafeedIds: rt.array(rt.string),
 });
+
+export const stopDatafeedsResponsePayloadRT = rt.record(
+  rt.string,
+  rt.type({
+    stopped: rt.boolean,
+  })
+);
diff --git a/...k/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts b/...k/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts
@@ -12,15 +12,19 @@ import { kfetch } from 'ui/kfetch';
 
 import { jobCustomSettingsRT } from './ml_api_types';
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
-import { getAllModuleJobIds } from '../../../../../common/log_analysis';
+import { getJobId } from '../../../../../common/log_analysis';
 
-export const callJobsSummaryAPI = async (spaceId: string, sourceId: string) => {
+export const callJobsSummaryAPI = async <JobType extends string>(
+  spaceId: string,
+  sourceId: string,
+  jobTypes: JobType[]
+) => {
   const response = await kfetch({
     method: 'POST',
     pathname: '/api/ml/jobs/jobs_summary',
     body: JSON.stringify(
       fetchJobStatusRequestPayloadRT.encode({
-        jobIds: getAllModuleJobIds(spaceId, sourceId),
+        jobIds: jobTypes.map(jobType => getJobId(spaceId, sourceId, jobType)),
       })
     ),
   });

diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts
@@ -12,7 +12,6 @@ import { kfetch } from 'ui/kfetch';
 
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 import { getJobIdPrefix } from '../../../../../common/log_analysis';
-import { jobCustomSettingsRT } from './ml_api_types';
 
 export const callSetupMlModuleAPI = async (
   moduleId: string,
@@ -21,8 +20,8 @@ export const callSetupMlModuleAPI = async (
   spaceId: string,
   sourceId: string,
   indexPattern: string,
-  timeField: string,
-  bucketSpan: number
+  jobOverrides: SetupMlModuleJobOverrides[] = [],
+  datafeedOverrides: SetupMlModuleDatafeedOverrides[] = []
 ) => {
   const response = await kfetch({
     method: 'POST',
@@ -34,25 +33,8 @@ export const callSetupMlModuleAPI = async (
         indexPatternName: indexPattern,
         prefix: getJobIdPrefix(spaceId, sourceId),
         startDatafeed: true,
-        jobOverrides: [
-          {
-            job_id: 'log-entry-rate' as const,
-            analysis_config: {
-              bucket_span: `${bucketSpan}ms`,
-            },
-            data_description: {
-              time_field: timeField,
-            },
-            custom_settings: {
-              logs_source_config: {
-                indexPattern,
-                timestampField: timeField,
-                bucketSpan,
-              },
-            },
-          },
-        ],
-        datafeedOverrides: [],
+        jobOverrides,
+        datafeedOverrides,
       })
     ),
   });
@@ -68,23 +50,20 @@ const setupMlModuleTimeParamsRT = rt.partial({
   end: rt.number,
 });
 
-const setupMlModuleLogEntryRateJobOverridesRT = rt.type({
-  job_id: rt.literal('log-entry-rate'),
-  analysis_config: rt.type({
-    bucket_span: rt.string,
-  }),
-  data_description: rt.type({
-    time_field: rt.string,
-  }),
-  custom_settings: jobCustomSettingsRT,
-});
+const setupMlModuleJobOverridesRT = rt.object;
+
+export type SetupMlModuleJobOverrides = rt.TypeOf<typeof setupMlModuleJobOverridesRT>;
+
+const setupMlModuleDatafeedOverridesRT = rt.object;
+
+export type SetupMlModuleDatafeedOverrides = rt.TypeOf<typeof setupMlModuleDatafeedOverridesRT>;
 
 const setupMlModuleRequestParamsRT = rt.type({
   indexPatternName: rt.string,
   prefix: rt.string,
   startDatafeed: rt.boolean,
-  jobOverrides: rt.array(setupMlModuleLogEntryRateJobOverridesRT),
-  datafeedOverrides: rt.array(rt.object),
+  jobOverrides: rt.array(setupMlModuleJobOverridesRT),
+  datafeedOverrides: rt.array(setupMlModuleDatafeedOverridesRT),
 });
 
 const setupMlModuleRequestPayloadRT = rt.intersection([

diff --git a/...g_analysis/api/index_patterns_validate.ts → ...logs/log_analysis/api/validate_indices.ts b/...g_analysis/api/index_patterns_validate.ts → ...logs/log_analysis/api/validate_indices.ts
@@ -10,20 +10,22 @@ import { identity } from 'fp-ts/lib/function';
 import { kfetch } from 'ui/kfetch';
 
 import {
-  LOG_ANALYSIS_VALIDATION_INDICES_PATH,
+  LOG_ANALYSIS_VALIDATE_INDICES_PATH,
+  ValidationIndicesFieldSpecification,
   validationIndicesRequestPayloadRT,
   validationIndicesResponsePayloadRT,
 } from '../../../../../common/http_api';
 
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 
-export const callIndexPatternsValidate = async (timestampField: string, indices: string[]) => {
+export const callValidateIndicesAPI = async (
+  indices: string[],
+  fields: ValidationIndicesFieldSpecification[]
+) => {
   const response = await kfetch({
     method: 'POST',
-    pathname: LOG_ANALYSIS_VALIDATION_INDICES_PATH,
-    body: JSON.stringify(
-      validationIndicesRequestPayloadRT.encode({ data: { timestampField, indices } })
-    ),
+    pathname: LOG_ANALYSIS_VALIDATE_INDICES_PATH,
+    body: JSON.stringify(validationIndicesRequestPayloadRT.encode({ data: { indices, fields } })),
   });
 
   return pipe(

diff --git a/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/index.ts b/x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/index.ts
@@ -6,7 +6,6 @@
 
 export * from './log_analysis_capabilities';
 export * from './log_analysis_cleanup';
-export * from './log_analysis_jobs';
-export * from './log_analysis_results';
-export * from './log_analysis_results_url_state';
-export * from './log_analysis_status_state';
+export * from './log_analysis_module';
+export * from './log_analysis_module_status';
+export * from './log_analysis_module_types';
diff --git a/...ck/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_capabilities.tsx b/...ck/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_capabilities.tsx
@@ -15,7 +15,7 @@ import { useTrackedPromise } from '../../../utils/use_tracked_promise';
 import {
   getMlCapabilitiesResponsePayloadRT,
   GetMlCapabilitiesResponsePayload,
-} from './ml_api_types';
+} from './api/ml_api_types';
 import { throwErrors, createPlainError } from '../../../../common/runtime_types';
 
 export const useLogAnalysisCapabilities = () => {