diff --git a/x-pack/plugins/ml/common/types/job_validation.ts b/x-pack/plugins/ml/common/types/job_validation.ts new file mode 100644 index 0000000000000..0c1db63ff3762 --- /dev/null +++ b/x-pack/plugins/ml/common/types/job_validation.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ErrorType } from '../util/errors'; + +export interface DatafeedValidationResponse { + valid: boolean; + documentsFound: boolean; + error?: ErrorType; +} diff --git a/x-pack/plugins/ml/common/types/saved_objects.ts b/x-pack/plugins/ml/common/types/saved_objects.ts index 0e48800dd845d..e376fddbe6272 100644 --- a/x-pack/plugins/ml/common/types/saved_objects.ts +++ b/x-pack/plugins/ml/common/types/saved_objects.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ErrorType } from '../util/errors'; +import type { ErrorType } from '../util/errors'; export type JobType = 'anomaly-detector' | 'data-frame-analytics'; export const ML_SAVED_OBJECT_TYPE = 'ml-job'; export const ML_MODULE_SAVED_OBJECT_TYPE = 'ml-module'; diff --git a/x-pack/plugins/ml/public/application/contexts/kibana/use_navigate_to_path.ts b/x-pack/plugins/ml/public/application/contexts/kibana/use_navigate_to_path.ts index 951d9d6dfded9..00050803b97c6 100644 --- a/x-pack/plugins/ml/public/application/contexts/kibana/use_navigate_to_path.ts +++ b/x-pack/plugins/ml/public/application/contexts/kibana/use_navigate_to_path.ts @@ -23,7 +23,7 @@ export const useNavigateToPath = () => { const location = useLocation(); return useCallback( - async (path: string | undefined, preserveSearch = false) => { + async (path: string | undefined, preserveSearch: boolean = false) => { if (path === undefined) return; const modifiedPath = `${path}${preserveSearch === true ? location.search : ''}`; /** diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts index a44b4bdef60c4..607a4fcf9a73c 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts @@ -502,6 +502,10 @@ export class JobCreator { return this._datafeed_config.indices; } + public set indices(indics: string[]) { + this._datafeed_config.indices = indics; + } + public get scriptFields(): Field[] { return this._scriptFields; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts index 78903e64686f5..46315ac3b02d8 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/util/general.ts @@ -258,17 +258,21 @@ export function convertToMultiMetricJob( jobCreator.createdBy = CREATED_BY_LABEL.MULTI_METRIC; jobCreator.modelPlot = false; stashJobForCloning(jobCreator, true, true); - navigateToPath(`jobs/new_job/${JOB_TYPE.MULTI_METRIC}`, true); } export function convertToAdvancedJob(jobCreator: JobCreatorType, navigateToPath: NavigateToPath) { jobCreator.createdBy = null; stashJobForCloning(jobCreator, true, true); - navigateToPath(`jobs/new_job/${JOB_TYPE.ADVANCED}`, true); } +export function resetAdvancedJob(jobCreator: JobCreatorType, navigateToPath: NavigateToPath) { + jobCreator.createdBy = null; + stashJobForCloning(jobCreator, true, false); + navigateToPath('/jobs/new_job'); +} + export function resetJob(jobCreator: JobCreatorType, navigateToPath: NavigateToPath) { jobCreator.jobId = ''; stashJobForCloning(jobCreator, true, true); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx index ce71cd80e45c0..9e5d1ac5eef6f 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/common/json_editor_flyout/json_editor_flyout.tsx @@ -204,7 +204,7 @@ export const JsonEditorFlyout: FC = ({ isDisabled, jobEditorMode, datafee > diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view.tsx new file mode 100644 index 0000000000000..c402ee4bf9799 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view.tsx @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState, useEffect, useCallback, useContext } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { i18n } from '@kbn/i18n'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiButtonEmpty, + EuiModal, + EuiButton, + EuiCallOut, + EuiSpacer, + EuiModalHeader, + EuiLoadingSpinner, + EuiModalHeaderTitle, + EuiModalBody, +} from '@elastic/eui'; + +import { JobCreatorContext } from '../../../job_creator_context'; +import { AdvancedJobCreator } from '../../../../../common/job_creator'; +import { resetAdvancedJob } from '../../../../../common/job_creator/util/general'; +import { + CombinedJob, + Datafeed, +} from '../../../../../../../../../common/types/anomaly_detection_jobs'; +import { extractErrorMessage } from '../../../../../../../../../common/util/errors'; +import type { DatafeedValidationResponse } from '../../../../../../../../../common/types/job_validation'; + +import { SavedObjectFinderUi } from '../../../../../../../../../../../../src/plugins/saved_objects/public'; +import { + useMlKibana, + useMlApiContext, + useNavigateToPath, +} from '../../../../../../../contexts/kibana'; + +const fixedPageSize: number = 8; + +enum STEP { + PICK_DATA_VIEW, + VALIDATE, +} + +interface Props { + onClose: () => void; +} + +export const ChangeDataViewModal: FC = ({ onClose }) => { + const { + services: { + savedObjects, + uiSettings, + data: { dataViews }, + }, + } = useMlKibana(); + const navigateToPath = useNavigateToPath(); + const { validateDatafeedPreview } = useMlApiContext(); + + const { jobCreator: jc } = useContext(JobCreatorContext); + const jobCreator = jc as AdvancedJobCreator; + + const [validating, setValidating] = useState(false); + const [step, setStep] = useState(STEP.PICK_DATA_VIEW); + + const [currentDataViewTitle, setCurrentDataViewTitle] = useState(''); + const [newDataViewTitle, setNewDataViewTitle] = useState(''); + const [validationResponse, setValidationResponse] = useState( + null + ); + + useEffect(function initialPageLoad() { + setCurrentDataViewTitle(jobCreator.indexPatternTitle); + }, []); + + useEffect( + function stepChange() { + if (step === STEP.PICK_DATA_VIEW) { + setValidationResponse(null); + } + }, + [step] + ); + + function onDataViewSelected(dataViewId: string) { + if (validating === false) { + setStep(STEP.VALIDATE); + validate(dataViewId); + } + } + + const validate = useCallback( + async (dataViewId: string) => { + setValidating(true); + + const { title } = await dataViews.get(dataViewId); + setNewDataViewTitle(title); + + const indices = title.split(','); + if (jobCreator.detectors.length) { + const datafeed: Datafeed = { ...jobCreator.datafeedConfig, indices }; + const resp = await validateDatafeedPreview({ + job: { + ...jobCreator.jobConfig, + datafeed_config: datafeed, + } as CombinedJob, + }); + setValidationResponse(resp); + } + setValidating(false); + }, + [dataViews, validateDatafeedPreview, jobCreator] + ); + + const applyDataView = useCallback(() => { + const newIndices = newDataViewTitle.split(','); + jobCreator.indices = newIndices; + resetAdvancedJob(jobCreator, navigateToPath); + }, [jobCreator, newDataViewTitle, navigateToPath]); + + return ( + <> + + + + + + + + + {step === STEP.PICK_DATA_VIEW && ( + <> + + + + + 'indexPatternApp', + name: i18n.translate( + 'xpack.ml.newJob.wizard.datafeedStep.dataView.step1.dataView', + { + defaultMessage: 'Index pattern', + } + ), + }, + ]} + fixedPageSize={fixedPageSize} + uiSettings={uiSettings} + savedObjects={savedObjects} + /> + + )} + {step === STEP.VALIDATE && ( + <> + + + + + {validating === true ? ( + <> + + + + ) : ( + + )} + + + + + + + + + + + applyDataView()} + isDisabled={validating} + data-test-subj="mlJobsImportButton" + > + + + + + + )} + + + + ); +}; + +const ValidationMessage: FC<{ + validationResponse: DatafeedValidationResponse | null; + dataViewTitle: string; +}> = ({ validationResponse, dataViewTitle }) => { + if (validationResponse === null) { + return ( + + + + ); + } + if (validationResponse.valid === true) { + if (validationResponse.documentsFound === true) { + return ( + + + + ); + } else { + return ( + + + + ); + } + } else { + return ( + + + + + + + + + + {validationResponse.error ? extractErrorMessage(validationResponse.error) : null} + + ); + } +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view_button.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view_button.tsx new file mode 100644 index 0000000000000..dc9af26236d8c --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/change_data_view_button.tsx @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { EuiButtonEmpty } from '@elastic/eui'; +import { Description } from './description'; +import { ChangeDataViewModal } from './change_data_view'; + +export const ChangeDataView: FC<{ isDisabled: boolean }> = ({ isDisabled }) => { + const [showFlyout, setShowFlyout] = useState(false); + + return ( + <> + {showFlyout && } + + + + + + + + ); +}; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/description.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/description.tsx new file mode 100644 index 0000000000000..2632660738a58 --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/description.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, FC } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui'; + +export const Description: FC = memo(({ children }) => { + const title = i18n.translate('xpack.ml.newJob.wizard.datafeedStep.dataView.title', { + defaultMessage: 'Index pattern', + }); + return ( + {title}} + description={ + + } + > + + <>{children} + + + ); +}); diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/index.ts b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/index.ts new file mode 100644 index 0000000000000..ef7c451b4889c --- /dev/null +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/components/data_view/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ChangeDataView } from './change_data_view_button'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/datafeed.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/datafeed.tsx index 77db2eb2419cd..47e488ab201ec 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/datafeed.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/datafeed_step/datafeed.tsx @@ -14,6 +14,7 @@ import { FrequencyInput } from './components/frequency'; import { ScrollSizeInput } from './components/scroll_size'; import { ResetQueryButton } from './components/reset_query'; import { TimeField } from './components/time_field'; +import { ChangeDataView } from './components/data_view'; import { WIZARD_STEPS, StepProps } from '../step_types'; import { JobCreatorContext } from '../job_creator_context'; import { JsonEditorFlyout, EDITOR_MODE } from '../common/json_editor_flyout'; @@ -46,6 +47,7 @@ export const DatafeedStep: FC = ({ setCurrentStep, isCurrentStep }) = + diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts index 883e5d499c3d4..720e54e386cbc 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/index.ts @@ -43,6 +43,7 @@ import type { FieldHistogramRequestConfig } from '../../datavisualizer/index_bas import type { DataRecognizerConfigResponse, Module } from '../../../../common/types/modules'; import { getHttp } from '../../util/dependency_cache'; import type { RuntimeMappings } from '../../../../common/types/fields'; +import type { DatafeedValidationResponse } from '../../../../common/types/job_validation'; export interface MlInfoResponse { defaults: MlServerDefaults; @@ -194,7 +195,7 @@ export function mlApiServicesProvider(httpService: HttpService) { }, validateJob(payload: { - job: Job; + job: CombinedJob; duration: { start?: number; end?: number; @@ -209,6 +210,15 @@ export function mlApiServicesProvider(httpService: HttpService) { }); }, + validateDatafeedPreview(payload: { job: CombinedJob }) { + const body = JSON.stringify(payload); + return httpService.http({ + path: `${basePath()}/validate/datafeed_preview`, + method: 'POST', + body, + }); + }, + validateCardinality$(job: CombinedJob): Observable { const body = JSON.stringify(job); return httpService.http$({ diff --git a/x-pack/plugins/ml/server/models/job_validation/index.ts b/x-pack/plugins/ml/server/models/job_validation/index.ts index 92d3e7d613efc..a527b9dcf3d4b 100644 --- a/x-pack/plugins/ml/server/models/job_validation/index.ts +++ b/x-pack/plugins/ml/server/models/job_validation/index.ts @@ -7,3 +7,7 @@ export { validateJob } from './job_validation'; export { validateCardinality } from './validate_cardinality'; +export { + validateDatafeedPreviewWithMessages, + validateDatafeedPreview, +} from './validate_datafeed_preview'; diff --git a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts index 838f188455d44..4cd2d8a95ee79 100644 --- a/x-pack/plugins/ml/server/models/job_validation/job_validation.ts +++ b/x-pack/plugins/ml/server/models/job_validation/job_validation.ts @@ -17,7 +17,7 @@ import { basicJobValidation, uniqWithIsEqual } from '../../../common/util/job_ut import { validateBucketSpan } from './validate_bucket_span'; import { validateCardinality } from './validate_cardinality'; import { validateInfluencers } from './validate_influencers'; -import { validateDatafeedPreview } from './validate_datafeed_preview'; +import { validateDatafeedPreviewWithMessages } from './validate_datafeed_preview'; import { validateModelMemoryLimit } from './validate_model_memory_limit'; import { validateTimeRange, isValidTimeField } from './validate_time_range'; import { validateJobSchema } from '../../routes/schemas/job_validation_schema'; @@ -111,7 +111,9 @@ export async function validateJob( validationMessages.push({ id: 'missing_summary_count_field_name' }); } - validationMessages.push(...(await validateDatafeedPreview(mlClient, authHeader, job))); + validationMessages.push( + ...(await validateDatafeedPreviewWithMessages(mlClient, authHeader, job)) + ); } else { validationMessages = basicValidation.messages; validationMessages.push({ id: 'skipped_extended_tests' }); diff --git a/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts b/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts index 4ae94229a930b..0775de7ae0e13 100644 --- a/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts +++ b/x-pack/plugins/ml/server/models/job_validation/validate_datafeed_preview.ts @@ -9,12 +9,25 @@ import type { MlClient } from '../../lib/ml_client'; import type { AuthorizationHeader } from '../../lib/request_authorization'; import type { CombinedJob } from '../../../common/types/anomaly_detection_jobs'; import type { JobValidationMessage } from '../../../common/constants/messages'; +import type { DatafeedValidationResponse } from '../../../common/types/job_validation'; -export async function validateDatafeedPreview( +export async function validateDatafeedPreviewWithMessages( mlClient: MlClient, authHeader: AuthorizationHeader, job: CombinedJob ): Promise { + const { valid, documentsFound } = await validateDatafeedPreview(mlClient, authHeader, job); + if (valid) { + return documentsFound ? [] : [{ id: 'datafeed_preview_no_documents' }]; + } + return [{ id: 'datafeed_preview_failed' }]; +} + +export async function validateDatafeedPreview( + mlClient: MlClient, + authHeader: AuthorizationHeader, + job: CombinedJob +): Promise { const { datafeed_config: datafeed, ...tempJob } = job; try { const { body } = (await mlClient.previewDatafeed( @@ -28,11 +41,15 @@ export async function validateDatafeedPreview( // previewDatafeed response type is incorrect )) as unknown as { body: unknown[] }; - if (Array.isArray(body) === false || body.length === 0) { - return [{ id: 'datafeed_preview_no_documents' }]; - } - return []; + return { + valid: true, + documentsFound: Array.isArray(body) && body.length > 0, + }; } catch (error) { - return [{ id: 'datafeed_preview_failed' }]; + return { + valid: false, + documentsFound: false, + error: error.body ?? error, + }; } } diff --git a/x-pack/plugins/ml/server/routes/apidoc.json b/x-pack/plugins/ml/server/routes/apidoc.json index 7f53ebb92b68a..226b69e06b48a 100644 --- a/x-pack/plugins/ml/server/routes/apidoc.json +++ b/x-pack/plugins/ml/server/routes/apidoc.json @@ -123,11 +123,13 @@ "GetJobAuditMessages", "GetAllJobAuditMessages", "ClearJobAuditMessages", + "JobValidation", "EstimateBucketSpan", "CalculateModelMemoryLimit", "ValidateCardinality", "ValidateJob", + "ValidateDataFeedPreview", "DatafeedService", "CreateDatafeed", diff --git a/x-pack/plugins/ml/server/routes/job_validation.ts b/x-pack/plugins/ml/server/routes/job_validation.ts index b75eab20e7bc0..bceb59fa33fc6 100644 --- a/x-pack/plugins/ml/server/routes/job_validation.ts +++ b/x-pack/plugins/ml/server/routes/job_validation.ts @@ -16,12 +16,18 @@ import { modelMemoryLimitSchema, validateCardinalitySchema, validateJobSchema, + validateDatafeedPreviewSchema, } from './schemas/job_validation_schema'; import { estimateBucketSpanFactory } from '../models/bucket_span_estimator'; import { calculateModelMemoryLimitProvider } from '../models/calculate_model_memory_limit'; -import { validateJob, validateCardinality } from '../models/job_validation'; +import { + validateJob, + validateCardinality, + validateDatafeedPreview, +} from '../models/job_validation'; import { getAuthorizationHeader } from '../lib/request_authorization'; import type { MlClient } from '../lib/ml_client'; +import { CombinedJob } from '../../common/types/anomaly_detection_jobs'; type CalculateModelMemoryLimitPayload = TypeOf; @@ -205,4 +211,40 @@ export function jobValidationRoutes({ router, mlLicense, routeGuard }: RouteInit } }) ); + + /** + * @apiGroup DataFeedPreviewValidation + * + * @api {post} /api/ml/validate/datafeed_preview Validates datafeed preview + * @apiName ValidateDataFeedPreview + * @apiDescription Validates that the datafeed preview runs successfully and produces results + * + * @apiSchema (body) validateDatafeedPreviewSchema + */ + router.post( + { + path: '/api/ml/validate/datafeed_preview', + validate: { + body: validateDatafeedPreviewSchema, + }, + options: { + tags: ['access:ml:canCreateJob'], + }, + }, + routeGuard.fullLicenseAPIGuard(async ({ client, mlClient, request, response }) => { + try { + const resp = await validateDatafeedPreview( + mlClient, + getAuthorizationHeader(request), + request.body.job as CombinedJob + ); + + return response.ok({ + body: resp, + }); + } catch (e) { + return response.customError(wrapError(e)); + } + }) + ); } diff --git a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts index a83bbbff6cec9..a481713f67359 100644 --- a/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts +++ b/x-pack/plugins/ml/server/routes/schemas/job_validation_schema.ts @@ -60,6 +60,13 @@ export const validateJobSchema = schema.object({ }), }); +export const validateDatafeedPreviewSchema = schema.object({ + job: schema.object({ + ...anomalyDetectionJobSchema, + datafeed_config: datafeedConfigSchema, + }), +}); + export const validateCardinalitySchema = schema.object({ ...anomalyDetectionJobSchema, datafeed_config: datafeedConfigSchema, diff --git a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts index 055b4b69ab7a6..e7ea71863352e 100644 --- a/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts +++ b/x-pack/test/api_integration/apis/ml/data_frame_analytics/delete.ts @@ -130,7 +130,7 @@ export default ({ getService }: FtrProviderContext) => { const destinationIndex = generateDestinationIndex(analyticsId); before(async () => { - await ml.api.createIndices(destinationIndex); + await ml.api.createIndex(destinationIndex); await ml.api.assertIndicesExist(destinationIndex); }); @@ -189,7 +189,7 @@ export default ({ getService }: FtrProviderContext) => { before(async () => { // Mimic real job by creating target index & index pattern after DFA job is created - await ml.api.createIndices(destinationIndex); + await ml.api.createIndex(destinationIndex); await ml.api.assertIndicesExist(destinationIndex); await ml.testResources.createIndexPatternIfNeeded(destinationIndex); }); diff --git a/x-pack/test/api_integration/apis/ml/job_validation/datafeed_preview_validation.ts b/x-pack/test/api_integration/apis/ml/job_validation/datafeed_preview_validation.ts new file mode 100644 index 0000000000000..c16050e08c886 --- /dev/null +++ b/x-pack/test/api_integration/apis/ml/job_validation/datafeed_preview_validation.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { estypes } from '@elastic/elasticsearch'; +import { FtrProviderContext } from '../../../ftr_provider_context'; +import { USER } from '../../../../functional/services/ml/security_common'; +import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/common_api'; + +const farequoteMappings: estypes.MappingTypeMapping = { + properties: { + '@timestamp': { + type: 'date', + }, + airline: { + type: 'keyword', + }, + responsetime: { + type: 'float', + }, + }, +}; + +function getBaseJobConfig() { + return { + job_id: 'test', + description: '', + analysis_config: { + bucket_span: '15m', + detectors: [ + { + function: 'mean', + field_name: 'responsetime', + }, + ], + influencers: [], + }, + analysis_limits: { + model_memory_limit: '11MB', + }, + data_description: { + time_field: '@timestamp', + time_format: 'epoch_ms', + }, + model_plot_config: { + enabled: false, + annotations_enabled: false, + }, + model_snapshot_retention_days: 10, + daily_model_snapshot_retention_after_days: 1, + allow_lazy_open: false, + datafeed_config: { + query: { + bool: { + must: [ + { + match_all: {}, + }, + ], + }, + }, + indices: ['ft_farequote'], + scroll_size: 1000, + delayed_data_check_config: { + enabled: true, + }, + job_id: 'test', + datafeed_id: 'datafeed-test', + }, + }; +} + +export default ({ getService }: FtrProviderContext) => { + const esArchiver = getService('esArchiver'); + const supertest = getService('supertestWithoutAuth'); + const ml = getService('ml'); + + describe('Validate datafeed preview', function () { + before(async () => { + await esArchiver.loadIfNeeded('x-pack/test/functional/es_archives/ml/farequote'); + await ml.testResources.setKibanaTimeZoneToUTC(); + await ml.api.createIndex('farequote_empty', farequoteMappings); + }); + + after(async () => { + await ml.api.cleanMlIndices(); + await ml.api.deleteIndices('farequote_empty'); + }); + + it(`should validate a job with documents`, async () => { + const job = getBaseJobConfig(); + + const { body } = await supertest + .post('/api/ml/validate/datafeed_preview') + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .send({ job }) + .expect(200); + + expect(body.valid).to.eql(true, `valid should be true, but got ${body.valid}`); + expect(body.documentsFound).to.eql( + true, + `documentsFound should be true, but got ${body.documentsFound}` + ); + }); + + it(`should fail to validate a job with documents and non-existent field`, async () => { + const job = getBaseJobConfig(); + job.analysis_config.detectors[0].field_name = 'no_such_field'; + + const { body } = await supertest + .post('/api/ml/validate/datafeed_preview') + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .send({ job }) + .expect(200); + + expect(body.valid).to.eql(false, `valid should be false, but got ${body.valid}`); + expect(body.documentsFound).to.eql( + false, + `documentsFound should be false, but got ${body.documentsFound}` + ); + }); + + it(`should validate a job with no documents`, async () => { + const job = getBaseJobConfig(); + job.datafeed_config.indices = ['farequote_empty']; + + const { body } = await supertest + .post('/api/ml/validate/datafeed_preview') + .auth(USER.ML_POWERUSER, ml.securityCommon.getPasswordForUser(USER.ML_POWERUSER)) + .set(COMMON_REQUEST_HEADERS) + .send({ job }) + .expect(200); + + expect(body.valid).to.eql(true, `valid should be true, but got ${body.valid}`); + expect(body.documentsFound).to.eql( + false, + `documentsFound should be false, but got ${body.documentsFound}` + ); + }); + + it(`should fail for viewer user`, async () => { + const job = getBaseJobConfig(); + + await supertest + .post('/api/ml/validate/datafeed_preview') + .auth(USER.ML_VIEWER, ml.securityCommon.getPasswordForUser(USER.ML_VIEWER)) + .set(COMMON_REQUEST_HEADERS) + .send({ job }) + .expect(403); + }); + + it(`should fail for unauthorized user`, async () => { + const job = getBaseJobConfig(); + + await supertest + .post('/api/ml/validate/datafeed_preview') + .auth(USER.ML_UNAUTHORIZED, ml.securityCommon.getPasswordForUser(USER.ML_UNAUTHORIZED)) + .set(COMMON_REQUEST_HEADERS) + .send({ job }) + .expect(403); + }); + }); +}; diff --git a/x-pack/test/api_integration/apis/ml/job_validation/index.ts b/x-pack/test/api_integration/apis/ml/job_validation/index.ts index 4b75102d7b0bf..be07ae3b1852a 100644 --- a/x-pack/test/api_integration/apis/ml/job_validation/index.ts +++ b/x-pack/test/api_integration/apis/ml/job_validation/index.ts @@ -13,5 +13,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./calculate_model_memory_limit')); loadTestFile(require.resolve('./cardinality')); loadTestFile(require.resolve('./validate')); + loadTestFile(require.resolve('./datafeed_preview_validation')); }); } diff --git a/x-pack/test/functional/services/ml/api.ts b/x-pack/test/functional/services/ml/api.ts index abde3bf365384..6ffd95f213c41 100644 --- a/x-pack/test/functional/services/ml/api.ts +++ b/x-pack/test/functional/services/ml/api.ts @@ -126,14 +126,20 @@ export function MachineLearningAPIProvider({ getService }: FtrProviderContext) { ); }, - async createIndices(indices: string) { + async createIndex( + indices: string, + mappings?: Record | estypes.MappingTypeMapping + ) { log.debug(`Creating indices: '${indices}'...`); if ((await es.indices.exists({ index: indices, allow_no_indices: false })).body === true) { log.debug(`Indices '${indices}' already exist. Nothing to create.`); return; } - const { body } = await es.indices.create({ index: indices }); + const { body } = await es.indices.create({ + index: indices, + ...(mappings ? { body: { mappings } } : {}), + }); expect(body) .to.have.property('acknowledged') .eql(true, 'Response for create request indices should be acknowledged.');