Add index subset selection to setup process

x-pack/legacy/plugins/infra/common/log_analysis/log_analysis.ts

@@ -54,3 +54,8 @@ export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
   ['skipped', 'hiddenAfterSuccess', 'skippedButReconfigurable', 'skippedButUpdatable'].includes(
     setupStatus
   );
+
+const KIBANA_SAMPLE_DATA_INDICES = ['kibana_sample_data_logs*'];
+
+export const isExampleDataIndex = (indexName: string) =>
+  KIBANA_SAMPLE_DATA_INDICES.includes(indexName);

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -16,7 +16,6 @@ import { useLogAnalysisCleanup } from './log_analysis_cleanup';
 import { useStatusState } from './log_analysis_status_state';
 
 const MODULE_ID = 'logs_ui_analysis';
-const SAMPLE_DATA_INDEX = 'kibana_sample_data_logs*';
 
 export const useLogAnalysisJobs = ({
   indexPattern,
@@ -29,11 +28,10 @@ export const useLogAnalysisJobs = ({
   spaceId: string;
   timeField: string;
 }) => {
-  const filteredIndexPattern = useMemo(() => removeSampleDataIndex(indexPattern), [indexPattern]);
   const { cleanupMLResources } = useLogAnalysisCleanup({ sourceId, spaceId });
   const [statusState, dispatch] = useStatusState({
     bucketSpan,
-    indexPattern: filteredIndexPattern,
+    indexPattern,
     timestampField: timeField,
   });
 
@@ -62,15 +60,19 @@ export const useLogAnalysisJobs = ({
   const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
     {
       cancelPreviousOn: 'resolution',
-      createPromise: async (start, end) => {
+      createPromise: async (
+        indices: string[],
+        start: number | undefined,
+        end: number | undefined
+      ) => {
         dispatch({ type: 'startedSetup' });
         return await callSetupMlModuleAPI(
           MODULE_ID,
           start,
           end,
           spaceId,
           sourceId,
-          filteredIndexPattern,
+          indices.join(','),
           timeField,
           bucketSpan
         );
@@ -82,7 +84,7 @@ export const useLogAnalysisJobs = ({
         dispatch({ type: 'failedSetup' });
       },
     },
-    [filteredIndexPattern, spaceId, sourceId, timeField, bucketSpan]
+    [spaceId, sourceId, timeField, bucketSpan]
   );
 
   const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
@@ -99,7 +101,7 @@ export const useLogAnalysisJobs = ({
         dispatch({ type: 'failedFetchingJobStatuses' });
       },
     },
-    [filteredIndexPattern, spaceId, sourceId]
+    [spaceId, sourceId]
   );
 
   const isLoadingSetupStatus = useMemo(
@@ -108,16 +110,18 @@ export const useLogAnalysisJobs = ({
     [fetchJobStatusRequest.state, fetchModuleDefinitionRequest.state]
   );
 
+  const availableIndices = useMemo(() => indexPattern.split(','), [indexPattern]);
+
   const viewResults = useCallback(() => {
     dispatch({ type: 'viewedResults' });
   }, []);
 
   const cleanupAndSetup = useCallback(
-    (start, end) => {
+    (indices: string[], start: number | undefined, end: number | undefined) => {
       dispatch({ type: 'startedSetup' });
       cleanupMLResources()
         .then(() => {
-          setupMlModule(start, end);
+          setupMlModule(indices, start, end);
         })
         .catch(() => {
           dispatch({ type: 'failedSetup' });
@@ -139,6 +143,7 @@ export const useLogAnalysisJobs = ({
   }, [fetchModuleDefinition]);
 
   return {
+    availableIndices,
     fetchJobStatus,
     isLoadingSetupStatus,
     jobStatus: statusState.jobStatus,
@@ -153,11 +158,3 @@ export const useLogAnalysisJobs = ({
 };
 
 export const LogAnalysisJobs = createContainer(useLogAnalysisJobs);
-//
-// This is needed due to: https://github.com/elastic/kibana/issues/43671
-const removeSampleDataIndex = (indexPattern: string) => {
-  return indexPattern
-    .split(',')
-    .filter(index => index !== SAMPLE_DATA_INDEX)
-    .join(',');
-};

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.tsx

@@ -3,35 +3,82 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { useState, useCallback } from 'react';
 
-type SetupHandler = (startTime?: number | undefined, endTime?: number | undefined) => void;
+import { useState, useCallback, useMemo } from 'react';
 
-interface Props {
+import { isExampleDataIndex } from '../../../../common/log_analysis';
+
+type SetupHandler = (
+  indices: string[],
+  startTime: number | undefined,
+  endTime: number | undefined
+) => void;
+
+interface AnalysisSetupStateArguments {
+  availableIndices: string[];
   cleanupAndSetupModule: SetupHandler;
   setupModule: SetupHandler;
 }
 
+type IndicesSelection = Record<string, boolean>;
+
+type ValidationErrors = 'TOO_FEW_SELECTED_INDICES';
+
 const fourWeeksInMs = 86400000 * 7 * 4;
 
-export const useAnalysisSetupState = ({ setupModule, cleanupAndSetupModule }: Props) => {
+export const useAnalysisSetupState = ({
+  availableIndices,
+  cleanupAndSetupModule,
+  setupModule,
+}: AnalysisSetupStateArguments) => {
   const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
   const [endTime, setEndTime] = useState<number | undefined>(undefined);
 
+  const [selectedIndices, setSelectedIndices] = useState<IndicesSelection>(
+    availableIndices.reduce(
+      (indexMap, indexName) => ({
+        ...indexMap,
+        [indexName]: !(availableIndices.length > 1 && isExampleDataIndex(indexName)),
+      }),
+      {}
+    )
+  );
+
+  const selectedIndexNames = useMemo(
+    () =>
+      Object.entries(selectedIndices)
+        .filter(([_indexName, isSelected]) => isSelected)
+        .map(([indexName]) => indexName),
+    [selectedIndices]
+  );
+
   const setup = useCallback(() => {
-    return setupModule(startTime, endTime);
-  }, [setupModule, startTime, endTime]);
+    return setupModule(selectedIndexNames, startTime, endTime);
+  }, [setupModule, selectedIndexNames, startTime, endTime]);
 
   const cleanupAndSetup = useCallback(() => {
-    return cleanupAndSetupModule(startTime, endTime);
-  }, [cleanupAndSetupModule, startTime, endTime]);
+    return cleanupAndSetupModule(selectedIndexNames, startTime, endTime);
+  }, [cleanupAndSetupModule, selectedIndexNames, startTime, endTime]);
+
+  const validationErrors: ValidationErrors[] = useMemo(
+    () => [
+      ...(Object.values(selectedIndices).some(isSelected => isSelected)
+        ? []
+        : ['TOO_FEW_SELECTED_INDICES' as const]),
+    ],
+    [selectedIndices]
+  );
 
   return {
     cleanupAndSetup,
     endTime,
+    selectedIndexNames,
+    selectedIndices,
     setEndTime,
+    setSelectedIndices,
     setStartTime,
     setup,
     startTime,
+    validationErrors,
   };
 };

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_status_state.tsx

@@ -348,11 +348,19 @@ const isJobConfigurationConsistent = (
       return (
         jobConfiguration &&
         jobConfiguration.bucketSpan === sourceConfiguration.bucketSpan &&
-        jobConfiguration.indexPattern === sourceConfiguration.indexPattern &&
+        jobConfiguration.indexPattern &&
+        isIndexPatternSubset(jobConfiguration.indexPattern, sourceConfiguration.indexPattern) &&
         jobConfiguration.timestampField === sourceConfiguration.timestampField
       );
     });
 
+const isIndexPatternSubset = (indexPatternSubset: string, indexPatternSuperset: string) => {
+  const subsetSubPatterns = indexPatternSubset.split(',');
+  const supersetSubPatterns = new Set(indexPatternSuperset.split(','));
+
+  return subsetSubPatterns.every(subPattern => supersetSubPatterns.has(subPattern));
+};
+
 export const useStatusState = (sourceConfiguration: JobSourceConfiguration) => {
   return useReducer(statusReducer, sourceConfiguration, createInitialState);
 };

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_content.tsx

@@ -17,12 +17,17 @@ import { AnalysisUnavailableContent } from './page_unavailable_content';
 import { AnalysisSetupStatusUnknownContent } from './page_setup_status_unknown';
 
 export const AnalysisPageContent = () => {
-  const { sourceId, source } = useContext(Source.Context);
+  const { sourceId } = useContext(Source.Context);
   const { hasLogAnalysisCapabilites } = useContext(LogAnalysisCapabilities.Context);
 
-  const { setup, cleanupAndSetup, setupStatus, viewResults, fetchJobStatus } = useContext(
-    LogAnalysisJobs.Context
-  );
+  const {
+    availableIndices,
+    cleanupAndSetup,
+    fetchJobStatus,
+    setup,
+    setupStatus,
+    viewResults,
+  } = useContext(LogAnalysisJobs.Context);
 
   useEffect(() => {
     fetchJobStatus();
@@ -50,10 +55,10 @@ export const AnalysisPageContent = () => {
   } else {
     return (
       <AnalysisSetupContent
+        availableIndices={availableIndices}
         setup={setup}
         cleanupAndSetup={cleanupAndSetup}
         setupStatus={setupStatus}
-        indexPattern={source ? source.configuration.logAlias : ''}
         viewResults={viewResults}
       />
     );

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_setup_content.tsx

@@ -22,19 +22,23 @@ import { SetupStatus } from '../../../../common/log_analysis';
 import { useTrackPageview } from '../../../hooks/use_track_metric';
 import { AnalysisSetupSteps } from './setup/steps';
 
-type SetupHandler = (startTime?: number | undefined, endTime?: number | undefined) => void;
+type SetupHandler = (
+  indices: string[],
+  startTime: number | undefined,
+  endTime: number | undefined
+) => void;
 
 interface AnalysisSetupContentProps {
+  availableIndices: string[];
   cleanupAndSetup: SetupHandler;
-  indexPattern: string;
   setup: SetupHandler;
   setupStatus: SetupStatus;
   viewResults: () => void;
 }
 
 export const AnalysisSetupContent: React.FunctionComponent<AnalysisSetupContentProps> = ({
+  availableIndices,
   cleanupAndSetup,
-  indexPattern,
   setup,
   setupStatus,
   viewResults,
@@ -71,10 +75,10 @@ export const AnalysisSetupContent: React.FunctionComponent<AnalysisSetupContentP
             </EuiText>
             <EuiSpacer />
             <AnalysisSetupSteps
+              availableIndices={availableIndices}
               setup={setup}
               cleanupAndSetup={cleanupAndSetup}
               viewResults={viewResults}
-              indexPattern={indexPattern}
               setupStatus={setupStatus}
             />
           </EuiPageContentBody>
@@ -86,7 +90,7 @@ export const AnalysisSetupContent: React.FunctionComponent<AnalysisSetupContentP
 
 // !important due to https://github.com/elastic/eui/issues/2232
 const AnalysisPageContent = euiStyled(EuiPageContent)`
-  max-width: 518px !important;
+  max-width: 768px !important;
 `;
 
 const AnalysisSetupPage = euiStyled(EuiPage)`

x-pack/legacy/plugins/infra/public/pages/logs/analysis/setup/analysis_setup_indices_form.tsx

@@ -0,0 +1,87 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { useMemo, useCallback } from 'react';
+
+import { i18n } from '@kbn/i18n';
+import { EuiCode, EuiDescribedFormGroup, EuiFormRow, EuiCheckboxGroup } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+const indicesSelectionLabel = i18n.translate('xpack.infra.analysisSetup.indicesSelectionLabel', {
+  defaultMessage: 'Indices',
+});
+
+type IndicesSelection = Record<string, boolean>;
+
+export const AnalysisSetupIndicesForm: React.FunctionComponent<{
+  indices: IndicesSelection;
+  onChangeSelectedIndices: (selectedIndices: IndicesSelection) => void;
+  validationErrors?: Array<'TOO_FEW_SELECTED_INDICES'>;
+}> = ({ indices, onChangeSelectedIndices, validationErrors = [] }) => {
+  const choices = useMemo(
+    () =>
+      Object.keys(indices).map(indexName => ({
+        id: indexName,
+        label: <EuiCode>{indexName}</EuiCode>,
+      })),
+    [indices]
+  );
+
+  const handleCheckboxGroupChange = useCallback(
+    indexName => {
+      onChangeSelectedIndices({
+        ...indices,
+        [indexName]: !indices[indexName],
+      });
+    },
+    [indices, onChangeSelectedIndices]
+  );
+
+  return (
+    <EuiDescribedFormGroup
+      idAria="indices"
+      title={
+        <FormattedMessage
+          id="xpack.infra.analysisSetup.indicesSelectionTitle"
+          defaultMessage="Choose indices"
+        />
+      }
+      description={
+        <FormattedMessage
+          id="xpack.infra.analysisSetup.indicesSelectionDescription"
+          defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyse a subset of the index names. Every selected index name must match at least one index with log entries."
+        />
+      }
+    >
+      <EuiFormRow
+        describedByIds={['indices']}
+        error={validationErrors.map(formatValidationError)}
+        fullWidth
+        isInvalid={validationErrors.length > 0}
+        label={indicesSelectionLabel}
+        labelType="legend"
+      >
+        <EuiCheckboxGroup
+          options={choices}
+          idToSelectedMap={indices}
+          onChange={handleCheckboxGroupChange}
+        />
+      </EuiFormRow>
+    </EuiDescribedFormGroup>
+  );
+};
+
+const formatValidationError = (validationError: 'TOO_FEW_SELECTED_INDICES') => {
+  switch (validationError) {
+    case 'TOO_FEW_SELECTED_INDICES':
+      return i18n.translate(
+        'xpack.infra.analysisSetup.indicesSelectionTooFewSelectedIndicesDescription',
+        {
+          defaultMessage: 'Select at least one index name.',
+        }
+      );
+  }
+};