diff --git a/docs/user/alerting/create-and-manage-rules.asciidoc b/docs/user/alerting/create-and-manage-rules.asciidoc index b796128b95bcb..7e35cd232566c 100644 --- a/docs/user/alerting/create-and-manage-rules.asciidoc +++ b/docs/user/alerting/create-and-manage-rules.asciidoc @@ -166,7 +166,8 @@ responses: Click the rule name to access a rule details page: [role="screenshot"] -image::images/rule-details-alerts-active.png[Rule details page with three alerts] +image::images/rule-details-alerts-active.png[Rule details page with multiple alerts] +// NOTE: This is an autogenerated screenshot. Do not edit it directly. In this example, the rule detects when a site serves more than a threshold number of bytes in a 24 hour period. Four sites are above the threshold. These are called alerts - occurrences of the condition being detected - and the alert name, status, time of detection, and duration of the condition are shown in this view. Alerts come and go from the list depending on whether the rule conditions are met. @@ -182,4 +183,4 @@ You can suppress future actions for a specific alert by turning on the *Mute* to [role="screenshot"] image::images/rule-details-disabling.png[Use the disable toggle to turn off rule checks and clear alerts tracked] - +// NOTE: This is an autogenerated screenshot. Do not edit it directly. diff --git a/docs/user/alerting/images/individual-enable-disable.png b/docs/user/alerting/images/individual-enable-disable.png index 60de6079befb8..dfac27dec39ee 100644 Binary files a/docs/user/alerting/images/individual-enable-disable.png and b/docs/user/alerting/images/individual-enable-disable.png differ diff --git a/docs/user/alerting/images/rule-details-alerts-active.png b/docs/user/alerting/images/rule-details-alerts-active.png index 978ee180223ba..deb2feff7993e 100644 Binary files a/docs/user/alerting/images/rule-details-alerts-active.png and b/docs/user/alerting/images/rule-details-alerts-active.png differ diff --git a/docs/user/alerting/images/rule-details-disabling.png b/docs/user/alerting/images/rule-details-disabling.png index bee157ad28564..ad74410ab14e0 100644 Binary files a/docs/user/alerting/images/rule-details-disabling.png and b/docs/user/alerting/images/rule-details-disabling.png differ diff --git a/docs/user/alerting/images/rules-ui.png b/docs/user/alerting/images/rules-ui.png index ba7b4db071fe2..250f46392a45f 100644 Binary files a/docs/user/alerting/images/rules-ui.png and b/docs/user/alerting/images/rules-ui.png differ diff --git a/docs/user/alerting/images/snooze-panel.png b/docs/user/alerting/images/snooze-panel.png index a65bfa6bf2e62..080c661ccccd6 100644 Binary files a/docs/user/alerting/images/snooze-panel.png and b/docs/user/alerting/images/snooze-panel.png differ diff --git a/package.json b/package.json index cc9081e57fb01..02aade28dfd77 100644 --- a/package.json +++ b/package.json @@ -1301,7 +1301,7 @@ "apidoc-markdown": "^7.2.4", "argsplit": "^1.0.5", "autoprefixer": "^10.4.7", - "axe-core": "^4.0.2", + "axe-core": "^4.6.1", "babel-jest": "^29.3.1", "babel-loader": "^8.2.5", "babel-plugin-add-module-exports": "^1.0.4", diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap index d86194ac7dbaf..37aeb88feefa8 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/__snapshots__/header.test.tsx.snap @@ -20,6 +20,21 @@ Array [ test - Elastic , + ,
`; + +exports[`SkipToMainContent renders 1`] = ` + +`; diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx index 227a94a208ca3..5449ee34bb662 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/header.tsx @@ -46,7 +46,7 @@ import { HeaderActionMenu } from './header_action_menu'; import { HeaderExtension } from './header_extension'; import { HeaderTopBanner } from './header_top_banner'; import { HeaderMenuButton } from './header_menu_button'; -import { ScreenReaderRouteAnnouncements } from './screen_reader_a11y'; +import { ScreenReaderRouteAnnouncements, SkipToMainContent } from './screen_reader_a11y'; export interface HeaderProps { kibanaVersion: string; @@ -114,6 +114,7 @@ export function Header({ customBranding$={customBranding$} appId$={application.currentAppId$} /> +
diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx index 3dfc25c93c25a..b468eefd51d79 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.test.tsx @@ -9,8 +9,8 @@ import React from 'react'; import { BehaviorSubject } from 'rxjs'; import { mountWithIntl } from '@kbn/test-jest-helpers'; -import { ScreenReaderRouteAnnouncements } from './screen_reader_a11y'; -import { mount } from 'enzyme'; +import { ScreenReaderRouteAnnouncements, SkipToMainContent } from './screen_reader_a11y'; +import { mount, render } from 'enzyme'; describe('ScreenReaderRouteAnnouncements', () => { it('renders', () => { @@ -67,3 +67,10 @@ describe('ScreenReaderRouteAnnouncements', () => { ).toBeTruthy(); }); }); + +describe('SkipToMainContent', () => { + it('renders', () => { + const component = render(); + expect(component).toMatchSnapshot(); + }); +}); diff --git a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx index 811da52341417..f879e896297dc 100644 --- a/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx +++ b/packages/core/chrome/core-chrome-browser-internal/src/ui/header/screen_reader_a11y.tsx @@ -8,7 +8,8 @@ import React, { FC, useState, useEffect } from 'react'; import useObservable from 'react-use/lib/useObservable'; -import { EuiScreenReaderLive } from '@elastic/eui'; +import { EuiScreenReaderLive, EuiSkipLink } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import type { InternalApplicationStart } from '@kbn/core-application-browser-internal'; import type { HeaderProps } from './header'; @@ -56,3 +57,25 @@ export const ScreenReaderRouteAnnouncements: FC<{ ); }; + +const fallbackContentQueries = [ + 'main', // Ideal target for all plugins using KibanaPageTemplate + '[role="main"]', // Fallback for plugins using deprecated EuiPageContent + '.kbnAppWrapper', // Last-ditch fallback for all plugins regardless of page template +]; + +export const SkipToMainContent = () => { + return ( + + {i18n.translate('core.ui.skipToMainButton', { + defaultMessage: 'Skip to main content', + })} + + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts index f93602bd99350..bfe48e43491a0 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/index.ts @@ -15,6 +15,7 @@ export { type IndexMapping, type IndexMappingMeta, type SavedObjectsTypeMappingDefinitions, + type IndexMappingMigrationStateMeta, } from './src/mappings'; export { SavedObjectsSerializer } from './src/serialization'; export { SavedObjectsTypeValidator } from './src/validation'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts index b7869bd12337d..7b2bb933fab3f 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/index.ts @@ -7,4 +7,9 @@ */ export { getTypes, getProperty, getRootProperties, getRootPropertiesObjects } from './lib'; -export type { SavedObjectsTypeMappingDefinitions, IndexMappingMeta, IndexMapping } from './types'; +export type { + SavedObjectsTypeMappingDefinitions, + IndexMappingMeta, + IndexMapping, + IndexMappingMigrationStateMeta, +} from './types'; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts index 0267f2ce27c1a..10faa1b03d31d 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/mappings/types.ts @@ -77,4 +77,19 @@ export interface IndexMappingMeta { * @remark: Only defined for indices using the zdt migration algorithm. */ docVersions?: { [k: string]: number }; + /** + * Info about the current state of the migration. + * Should only be present if a migration is in progress or was interrupted. + * + * @remark: Only defined for indices using the zdt migration algorithm. + */ + migrationState?: IndexMappingMigrationStateMeta; +} + +/** @internal */ +export interface IndexMappingMigrationStateMeta { + /** + * Indicates that the algorithm is currently converting the documents. + */ + convertingDocuments: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts index 8e7816a12fb53..01fc57d46462b 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/model_version/model_version_from_mappings.ts @@ -10,16 +10,21 @@ import type { IndexMapping, IndexMappingMeta } from '../mappings'; import type { ModelVersionMap } from './version_map'; import { assertValidModelVersion } from './conversion'; +export interface GetModelVersionsFromMappingsOpts { + mappings: IndexMapping; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings. */ export const getModelVersionsFromMappings = ({ mappings, source, -}: { - mappings: IndexMapping; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingsOpts): ModelVersionMap | undefined => { if (!mappings._meta) { return undefined; } @@ -27,25 +32,35 @@ export const getModelVersionsFromMappings = ({ return getModelVersionsFromMappingMeta({ meta: mappings._meta, source, + knownTypes, }); }; +export interface GetModelVersionsFromMappingMetaOpts { + meta: IndexMappingMeta; + source: 'mappingVersions' | 'docVersions'; + /** if specified, will filter the types with the provided list */ + knownTypes?: string[]; +} + /** * Build the version map from the specified source of the provided mappings meta. */ export const getModelVersionsFromMappingMeta = ({ meta, source, -}: { - meta: IndexMappingMeta; - source: 'mappingVersions' | 'docVersions'; -}): ModelVersionMap | undefined => { + knownTypes, +}: GetModelVersionsFromMappingMetaOpts): ModelVersionMap | undefined => { const indexVersions = source === 'mappingVersions' ? meta.mappingVersions : meta.docVersions; if (!indexVersions) { return undefined; } + const typeSet = knownTypes ? new Set(knownTypes) : undefined; + return Object.entries(indexVersions).reduce((map, [type, rawVersion]) => { - map[type] = assertValidModelVersion(rawVersion); + if (!typeSet || typeSet.has(type)) { + map[type] = assertValidModelVersion(rawVersion); + } return map; }, {}); }; diff --git a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts index c4d48d66a6a06..6d1c0e3df4ce0 100644 --- a/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts +++ b/packages/core/saved-objects/core-saved-objects-base-server-internal/src/saved_objects_config.ts @@ -32,6 +32,9 @@ const migrationSchema = schema.object({ pollInterval: schema.number({ defaultValue: 1_500 }), skip: schema.boolean({ defaultValue: false }), retryAttempts: schema.number({ defaultValue: 15 }), + zdt: schema.object({ + metaPickupSyncDelaySec: schema.number({ min: 1, defaultValue: 120 }), + }), }); export type SavedObjectsMigrationConfigType = TypeOf; @@ -60,6 +63,7 @@ export const savedObjectsConfig: ServiceConfigDescriptor path: 'savedObjects', schema: soSchema, }; + export class SavedObjectConfig { public maxImportPayloadBytes: number; public maxImportExportSize: number; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts index 5dfdb05a0bca8..8b8e0f830261d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/constants.ts @@ -7,3 +7,5 @@ */ export const CLUSTER_SHARD_LIMIT_EXCEEDED_REASON = `[cluster_shard_limit_exceeded] Upgrading Kibana requires adding a small number of new shards. Ensure that Kibana is able to add 10 more shards by increasing the cluster.max_shards_per_node setting, or removing indices to clear up resources.`; + +export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts new file mode 100644 index 0000000000000..1181eb992be35 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/common/redact_state.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { BulkOperation } from '../model/create_batches'; + +export const redactBulkOperationBatches = ( + bulkOperationBatches: BulkOperation[][] +): BulkOperationContainer[][] => { + return bulkOperationBatches.map((batch) => + batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) + ); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts index 22c58ae62adcb..c83cab1c1d0bc 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/migrate_raw_docs.ts @@ -15,8 +15,8 @@ import type { SavedObjectSanitizedDoc, SavedObjectsRawDoc, SavedObjectUnsanitizedDoc, + ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import { SavedObjectsSerializer } from '@kbn/core-saved-objects-base-server-internal'; import type { MigrateAndConvertFn } from '../document_migrator/document_migrator'; import { TransformSavedObjectDocumentError } from '.'; @@ -65,7 +65,7 @@ export class CorruptSavedObjectError extends Error { * @returns {SavedObjectsRawDoc[]} */ export async function migrateRawDocs( - serializer: SavedObjectsSerializer, + serializer: ISavedObjectsSerializer, migrateDoc: MigrateAndConvertFn, rawDocs: SavedObjectsRawDoc[] ): Promise { @@ -86,7 +86,7 @@ export async function migrateRawDocs( } interface MigrateRawDocsSafelyDeps { - serializer: SavedObjectsSerializer; + serializer: ISavedObjectsSerializer; migrateDoc: MigrateAndConvertFn; rawDocs: SavedObjectsRawDoc[]; } @@ -181,7 +181,7 @@ function transformNonBlocking( async function migrateMapToRawDoc( migrateMethod: MigrateFn, savedObject: SavedObjectSanitizedDoc, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): Promise { return [...(await migrateMethod(savedObject))].map((attrs) => serializer.savedObjectToRaw({ @@ -201,7 +201,7 @@ async function migrateMapToRawDoc( function convertToRawAddMigrationVersion( rawDoc: SavedObjectsRawDoc, options: { namespaceTreatment: 'lax' }, - serializer: SavedObjectsSerializer + serializer: ISavedObjectsSerializer ): SavedObjectSanitizedDoc { const savedObject = serializer.rawToSavedObject(rawDoc, options); if (!savedObject.migrationVersion && !savedObject.typeMigrationVersion) { diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts index f5a462ebcedac..e6855a1256b54 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/kibana_migrator.test.ts @@ -288,6 +288,9 @@ const mockOptions = () => { scrollDuration: '10m', skip: false, retryAttempts: 20, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, client: mockedClient, docLinks: docLinksServiceMock.createSetupContract(), diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts index 48a3bad0d0960..d03b4f7378da4 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.test.ts @@ -51,6 +51,9 @@ describe('migrationsStateActionMachine', () => { scrollDuration: '0s', skip: false, retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, }, typeRegistry, docLinks, diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts index 1b5caf3c4e75d..92b0054bd47c3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/migrations_state_action_machine.ts @@ -15,12 +15,11 @@ import { getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; -import type { BulkOperationContainer } from '@elastic/elasticsearch/lib/api/types'; import { logActionResponse, logStateTransition } from './common/utils/logs'; import { type Model, type Next, stateActionMachine } from './state_action_machine'; import { cleanup } from './migrations_state_machine_cleanup'; import type { ReindexSourceToTempTransform, ReindexSourceToTempIndexBulk, State } from './state'; -import type { BulkOperation } from './model/create_batches'; +import { redactBulkOperationBatches } from './common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -159,11 +158,3 @@ export async function migrationStateActionMachine({ } } } - -const redactBulkOperationBatches = ( - bulkOperationBatches: BulkOperation[][] -): BulkOperationContainer[][] => { - return bulkOperationBatches.map((batch) => - batch.map((operation) => (Array.isArray(operation) ? operation[0] : operation)) - ); -}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts index ec19f834d9ceb..008d074b2cd6f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/create_batches.ts @@ -14,6 +14,7 @@ import type { TransformErrorObjects } from '../core'; export type BulkIndexOperationTuple = [BulkOperationContainer, SavedObjectsRawDocSource]; export type BulkOperation = BulkIndexOperationTuple | BulkOperationContainer; +export type BulkOperationBatch = BulkOperation[]; export interface CreateBatchesParams { documents: SavedObjectsRawDoc[]; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts index 2f78d8745e55a..23ddee5043261 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/model/model.ts @@ -46,9 +46,10 @@ import { } from './helpers'; import { createBatches } from './create_batches'; import type { MigrationLog } from '../types'; -import { CLUSTER_SHARD_LIMIT_EXCEEDED_REASON } from '../common/constants'; - -export const FATAL_REASON_REQUEST_ENTITY_TOO_LARGE = `While indexing a batch of saved objects, Elasticsearch returned a 413 Request Entity Too Large exception. Ensure that the Kibana configuration option 'migrations.maxBatchSizeBytes' is set to a value that is lower than or equal to the Elasticsearch 'http.max_content_length' configuration option.`; +import { + CLUSTER_SHARD_LIMIT_EXCEEDED_REASON, + FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, +} from '../common/constants'; export const model = (currentState: State, resW: ResponseType): State => { // The action response `resW` is weakly typed, the type includes all action diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts index bb135c115ce92..a3db45a3748cc 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/index.ts @@ -6,17 +6,7 @@ * Side Public License, v 1. */ -import type { - IncompatibleClusterRoutingAllocation, - RetryableEsClientError, - WaitForTaskCompletionTimeout, - IndexNotYellowTimeout, - IndexNotGreenTimeout, - ClusterShardLimitExceeded, - IndexNotFound, - AliasNotFound, - IncompatibleMappingException, -} from '../../actions'; +import type { ActionErrorTypeMap as BaseActionErrorTypeMap } from '../../actions'; export { initAction as init, @@ -25,7 +15,16 @@ export { updateAliases, updateMappings, updateAndPickupMappings, + cleanupUnknownAndExcluded, + waitForDeleteByQueryTask, waitForPickupUpdatedMappingsTask, + refreshIndex, + openPit, + readWithPit, + closePit, + transformDocs, + bulkOverwriteTransformedDocuments, + noop, type InitActionParams, type IncompatibleClusterRoutingAllocation, type RetryableEsClientError, @@ -33,17 +32,11 @@ export { type IndexNotFound, } from '../../actions'; -export interface ActionErrorTypeMap { - wait_for_task_completion_timeout: WaitForTaskCompletionTimeout; - incompatible_cluster_routing_allocation: IncompatibleClusterRoutingAllocation; - retryable_es_client_error: RetryableEsClientError; - index_not_found_exception: IndexNotFound; - index_not_green_timeout: IndexNotGreenTimeout; - index_not_yellow_timeout: IndexNotYellowTimeout; - cluster_shard_limit_exceeded: ClusterShardLimitExceeded; - alias_not_found_exception: AliasNotFound; - incompatible_mapping_exception: IncompatibleMappingException; -} +export { updateIndexMeta, type UpdateIndexMetaParams } from './update_index_meta'; +export { waitForDelay, type WaitForDelayParams } from './wait_for_delay'; + +// alias in case we need to extend it with zdt specific actions/errors +export type ActionErrorTypeMap = BaseActionErrorTypeMap; /** Type guard for narrowing the type of a left */ export function isTypeof( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts new file mode 100644 index 0000000000000..8f209365a285e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const updateMappingsMock = jest.fn(); + +jest.doMock('../../actions/update_mappings', () => { + const actual = jest.requireActual('../../actions/update_mappings'); + return { + ...actual, + updateMappings: updateMappingsMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts new file mode 100644 index 0000000000000..6ed55ccb49ebe --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.test.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { updateMappingsMock } from './update_index_meta.test.mocks'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateIndexMeta } from './update_index_meta'; + +describe('updateIndexMeta', () => { + it('calls updateMappings with the correct parameters', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = { + mappingVersions: { + foo: 1, + bar: 1, + }, + }; + + updateIndexMeta({ client, index, meta }); + + expect(updateMappingsMock).toHaveBeenCalledTimes(1); + expect(updateMappingsMock).toHaveBeenCalledWith({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); + }); + + it('returns the response from updateMappings', () => { + const client = elasticsearchClientMock.createElasticsearchClient(); + const index = '.kibana_1'; + const meta: IndexMappingMeta = {}; + + const expected = Symbol(); + updateMappingsMock.mockReturnValue(expected); + + const actual = updateIndexMeta({ client, index, meta }); + + expect(actual).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts new file mode 100644 index 0000000000000..195e282dce5aa --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/update_index_meta.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; +import { updateMappings } from '../../actions'; + +export interface UpdateIndexMetaParams { + client: ElasticsearchClient; + index: string; + meta: IndexMappingMeta; +} + +export const updateIndexMeta = ({ + client, + index, + meta, +}: UpdateIndexMetaParams): ReturnType => { + return updateMappings({ + client, + index, + mappings: { + properties: {}, + _meta: meta, + }, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts new file mode 100644 index 0000000000000..fbfc144bd8150 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { waitForDelay } from './wait_for_delay'; + +const nextTick = () => new Promise((resolve) => resolve()); +const aFewTicks = () => nextTick().then(nextTick).then(nextTick); + +describe('waitForDelay', () => { + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + it('resolves after the specified amount of time', async () => { + const handler = jest.fn(); + + waitForDelay({ delayInSec: 5 })().then(handler); + + expect(handler).not.toHaveBeenCalled(); + + jest.advanceTimersByTime(5000); + await aFewTicks(); + + expect(handler).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts new file mode 100644 index 0000000000000..302a702331de4 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/actions/wait_for_delay.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import * as TaskEither from 'fp-ts/lib/TaskEither'; + +export interface WaitForDelayParams { + delayInSec: number; +} + +export const waitForDelay = ({ + delayInSec, +}: WaitForDelayParams): TaskEither.TaskEither => { + return () => { + return delay(delayInSec) + .then(() => Either.right('wait_succeeded' as const)) + .catch((err) => { + // will never happen + throw err; + }); + }; +}; + +const delay = (delayInSec: number) => + new Promise((resolve) => setTimeout(resolve, delayInSec * 1000)); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts index 7a660ea470443..c31d4bc799b3b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/create_context.ts @@ -21,12 +21,15 @@ export const createContext = ({ types, docLinks, migrationConfig, + documentMigrator, elasticsearchClient, indexPrefix, typeRegistry, serializer, }: CreateContextOps): MigratorContext => { return { + migrationConfig, + documentMigrator, kibanaVersion, indexPrefix, types, @@ -37,5 +40,6 @@ export const createContext = ({ maxRetryAttempts: migrationConfig.retryAttempts, migrationDocLinks: docLinks.links.kibanaUpgradeSavedObjects, deletedTypes: REMOVED_TYPES, + discardCorruptObjects: Boolean(migrationConfig.discardCorruptObjects), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts index 5b6d4b2fe27e9..95ca7282daf57 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/context/types.ts @@ -11,13 +11,19 @@ import type { ISavedObjectTypeRegistry, ISavedObjectsSerializer, } from '@kbn/core-saved-objects-server'; -import type { ModelVersionMap } from '@kbn/core-saved-objects-base-server-internal'; +import type { + ModelVersionMap, + SavedObjectsMigrationConfigType, +} from '@kbn/core-saved-objects-base-server-internal'; import type { DocLinks } from '@kbn/doc-links'; +import { VersionedTransformer } from '../../document_migrator'; /** * The set of static, precomputed values and services used by the ZDT migration */ export interface MigratorContext { + /** The migration configuration */ + readonly migrationConfig: SavedObjectsMigrationConfigType; /** The current Kibana version */ readonly kibanaVersion: string; /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ @@ -34,8 +40,12 @@ export interface MigratorContext { readonly migrationDocLinks: DocLinks['kibanaUpgradeSavedObjects']; /** SO serializer to use for migration */ readonly serializer: ISavedObjectsSerializer; + /** The doc migrator to use */ + readonly documentMigrator: VersionedTransformer; /** The SO type registry to use for the migration */ readonly typeRegistry: ISavedObjectTypeRegistry; /** List of types that are no longer registered */ readonly deletedTypes: string[]; + /** If true, corrupted objects will be discarded instead of failing the migration */ + readonly discardCorruptObjects: boolean; } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts index 8982a1a9c6c7e..cb6a30ce50c6f 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/migration_state_action_machine.ts @@ -12,11 +12,17 @@ import { getErrorMessage, getRequestDebugMeta, } from '@kbn/core-elasticsearch-client-server-internal'; +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; import { logStateTransition, logActionResponse } from '../common/utils'; import { type Next, stateActionMachine } from '../state_action_machine'; import { cleanup } from '../migrations_state_machine_cleanup'; -import type { State } from './state'; +import type { + State, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, +} from './state'; import type { MigratorContext } from './context'; +import { redactBulkOperationBatches } from '../common/redact_state'; /** * A specialized migrations-specific state-action machine that: @@ -60,23 +66,12 @@ export async function migrationStateActionMachine({ // the _id's of documents const redactedNewState = { ...newState, - /* TODO: commented until we have model stages that process outdated docs. (attrs not on model atm) - ...{ - outdatedDocuments: ( - (newState as ReindexSourceToTempTransform).outdatedDocuments ?? [] - ).map( - (doc) => - ({ - _id: doc._id, - } as SavedObjectsRawDoc) - ), - }, - ...{ - transformedDocBatches: ( - (newState as ReindexSourceToTempIndexBulk).transformedDocBatches ?? [] - ).map((batches) => batches.map((doc) => ({ _id: doc._id }))) as [SavedObjectsRawDoc[]], - }, - */ + outdatedDocuments: ( + (newState as OutdatedDocumentsSearchTransformState).outdatedDocuments ?? [] + ).map((doc) => ({ _id: doc._id } as SavedObjectsRawDoc)), + bulkOperationBatches: redactBulkOperationBatches( + (newState as OutdatedDocumentsSearchBulkIndexState).bulkOperationBatches ?? [[]] + ), }; const now = Date.now(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts index c0316b954e5f3..bef92e1986748 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.test.ts @@ -6,13 +6,13 @@ * Side Public License, v 1. */ -import { StageMocks } from './model.test.mocks'; +import './model.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { createContextMock, MockedMigratorContext } from '../test_helpers'; import type { RetryableEsClientError } from '../../actions'; import type { State, BaseState, FatalState, AllActionStates } from '../state'; import type { StateActionResponse } from './types'; -import { model } from './model'; +import { model, modelStageMap } from './model'; describe('model', () => { let context: MockedMigratorContext; @@ -128,16 +128,7 @@ describe('model', () => { }, }); - const stageMapping: Record = { - INIT: StageMocks.init, - CREATE_TARGET_INDEX: StageMocks.createTargetIndex, - UPDATE_INDEX_MAPPINGS: StageMocks.updateIndexMappings, - UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: StageMocks.updateIndexMappingsWaitForTask, - UPDATE_MAPPING_MODEL_VERSIONS: StageMocks.updateMappingModelVersion, - UPDATE_ALIASES: StageMocks.updateAliases, - }; - - Object.entries(stageMapping).forEach(([stage, handler]) => { + Object.entries(modelStageMap).forEach(([stage, handler]) => { test(`dispatch ${stage} state`, () => { const state = createStubState(stage as AllActionStates); const res = createStubResponse(); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts index 62971c3a614aa..483e97e3a7f85 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/model.ts @@ -12,9 +12,44 @@ import type { ResponseType } from '../next'; import { delayRetryState, resetRetryState } from '../../model/retry_state'; import { throwBadControlState } from '../../model/helpers'; import { isTypeof } from '../actions'; -import { MigratorContext } from '../context'; +import type { MigratorContext } from '../context'; +import type { ModelStage } from './types'; import * as Stages from './stages'; -import { StateActionResponse } from './types'; + +type ModelStageMap = { + [K in AllActionStates]: ModelStage; +}; + +type AnyModelStageHandler = ( + state: State, + response: Either.Either, + ctx: MigratorContext +) => State; + +export const modelStageMap: ModelStageMap = { + INIT: Stages.init, + CREATE_TARGET_INDEX: Stages.createTargetIndex, + UPDATE_ALIASES: Stages.updateAliases, + UPDATE_INDEX_MAPPINGS: Stages.updateIndexMappings, + UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: Stages.updateIndexMappingsWaitForTask, + UPDATE_MAPPING_MODEL_VERSIONS: Stages.updateMappingModelVersion, + INDEX_STATE_UPDATE_DONE: Stages.indexStateUpdateDone, + DOCUMENTS_UPDATE_INIT: Stages.documentsUpdateInit, + SET_DOC_MIGRATION_STARTED: Stages.setDocMigrationStarted, + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: Stages.setDocMigrationStartedWaitForInstances, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: Stages.cleanupUnknownAndExcludedDocs, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: Stages.cleanupUnknownAndExcludedDocsWaitForTask, + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: Stages.cleanupUnknownAndExcludedDocsRefresh, + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: Stages.outdatedDocumentsSearchOpenPit, + OUTDATED_DOCUMENTS_SEARCH_READ: Stages.outdatedDocumentsSearchRead, + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: Stages.outdatedDocumentsSearchTransform, + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: Stages.outdatedDocumentsSearchBulkIndex, + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: Stages.outdatedDocumentsSearchClosePit, + OUTDATED_DOCUMENTS_SEARCH_REFRESH: Stages.outdatedDocumentsSearchRefresh, + UPDATE_DOCUMENT_MODEL_VERSIONS: Stages.updateDocumentModelVersion, + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: + Stages.updateDocumentModelVersionWaitForInstances, +}; export const model = ( current: State, @@ -29,44 +64,14 @@ export const model = ( current = resetRetryState(current); } - switch (current.controlState) { - case 'INIT': - return Stages.init(current, response as StateActionResponse<'INIT'>, context); - case 'CREATE_TARGET_INDEX': - return Stages.createTargetIndex( - current, - response as StateActionResponse<'CREATE_TARGET_INDEX'>, - context - ); - case 'UPDATE_ALIASES': - return Stages.updateAliases( - current, - response as StateActionResponse<'UPDATE_ALIASES'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS': - return Stages.updateIndexMappings( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context - ); - case 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK': - return Stages.updateIndexMappingsWaitForTask( - current, - response as StateActionResponse<'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'>, - context - ); - case 'UPDATE_MAPPING_MODEL_VERSIONS': - return Stages.updateMappingModelVersion( - current, - response as StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'>, - context - ); - case 'DONE': - case 'FATAL': - // The state-action machine will never call the model in the terminating states - return throwBadControlState(current as never); - default: - return throwBadControlState(current); + if (current.controlState === 'DONE' || current.controlState === 'FATAL') { + return throwBadControlState(current as never); + } + + const stageHandler = modelStageMap[current.controlState] as AnyModelStageHandler; + if (!stageHandler) { + return throwBadControlState(current as never); } + + return stageHandler(current, response, context); }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts new file mode 100644 index 0000000000000..29730ab1ec789 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; + +describe('Stage: cleanupUnknownAndExcludedDocs', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK when successful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.right({ + type: 'cleanup_started', + taskId: '42', + errorsByType: {}, + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> FATAL when unsuccessful', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'> = Either.left({ + type: 'unknown_docs_found', + unknownDocs: [], + }); + + const newState = cleanupUnknownAndExcludedDocs(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts new file mode 100644 index 0000000000000..6aca449a73de5 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { extractUnknownDocFailureReason } from '../../../model/extract_errors'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocs: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + return { + ...state, + controlState: 'FATAL', + reason: extractUnknownDocFailureReason( + context.migrationDocLinks.resolveMigrationFailures, + res.left.unknownDocs + ), + }; + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: res.right.taskId, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts new file mode 100644 index 0000000000000..10d7ecc6a3841 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; + +describe('Stage: cleanupUnknownAndExcludedDocsRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsRefreshState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful', () => { + const state = createState(); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'>; + + const newState = cleanupUnknownAndExcludedDocsRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts new file mode 100644 index 0000000000000..32c4dc5bf6634 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsRefresh: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts new file mode 100644 index 0000000000000..e6deae5590770 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { CleanupUnknownAndExcludedDocsWaitForTaskState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; + +describe('Stage: cleanupUnknownAndExcludedDocsWaitForTask', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): CleanupUnknownAndExcludedDocsWaitForTaskState => ({ + ...createPostDocInitState(), + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + deleteTaskId: '42', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK in case of wait_for_task_completion_timeout', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'wait_for_task_completion_timeout', + message: 'woups', + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS in case of cleanup_failed', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount: 1, + retryDelay: expect.any(Number), + logs: expect.any(Array), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> FATAL in case of cleanup_failed when exceeding retry count', () => { + const state = createState({ + retryCount: context.maxRetryAttempts + 1, + }); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = Either.left( + { + type: 'cleanup_failed', + failures: [], + versionConflicts: 42, + } + ); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH when successful and docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 9000, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }); + }); + + it('CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT when successful and no docs were deleted', () => { + const state = createState(); + const res: StateActionResponse<'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'> = + Either.right({ + type: 'cleanup_successful', + deleted: 0, + }); + + const newState = cleanupUnknownAndExcludedDocsWaitForTask(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts new file mode 100644 index 0000000000000..83baaccea9921 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/cleanup_unknown_and_excluded_docs_wait_for_task.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { delayRetryState } from '../../../model/retry_state'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const cleanupUnknownAndExcludedDocsWaitForTask: ModelStage< + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK', + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH' + | 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + | 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'wait_for_task_completion_timeout')) { + // After waiting for the specified timeout, the task has not yet + // completed. Retry this step to see if the task has completed after an + // exponential delay. We will basically keep polling forever until the + // Elasticsearch task succeeds or fails. + return delayRetryState(state, res.left.message, Number.MAX_SAFE_INTEGER); + } else { + if (state.retryCount < context.maxRetryAttempts) { + const retryCount = state.retryCount + 1; + const retryDelay = 1500 + 1000 * Math.random(); + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + hasDeletedDocs: true, + retryCount, + retryDelay, + logs: [ + ...state.logs, + { + level: 'warning', + message: `Errors occurred whilst deleting unwanted documents. Retrying attempt ${retryCount}.`, + }, + ], + }; + } else { + const failures = res.left.failures.length; + const versionConflicts = res.left.versionConflicts ?? 0; + let reason = `Migration failed because it was unable to delete unwanted documents from the ${state.currentIndex} system index (${failures} failures and ${versionConflicts} conflicts)`; + if (failures) { + reason += `:\n` + res.left.failures.map((failure: string) => `- ${failure}\n`).join(''); + } + return { + ...state, + controlState: 'FATAL', + reason, + }; + } + } + } + + const mustRefresh = + state.hasDeletedDocs || typeof res.right.deleted === 'undefined' || res.right.deleted > 0; + + if (mustRefresh) { + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH', + }; + } else { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts index cd15594d32b29..4aaee33a8f974 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.test.ts @@ -84,7 +84,7 @@ describe('Stage: createTargetIndex', () => { }); }); - it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful', () => { + it('CREATE_TARGET_INDEX -> UPDATE_ALIASES when successful and alias actions are not empty', () => { const state = createState(); const res: StateActionResponse<'CREATE_TARGET_INDEX'> = Either.right('create_index_succeeded'); @@ -101,6 +101,27 @@ describe('Stage: createTargetIndex', () => { currentIndexMeta: state.indexMappings._meta, aliases: [], aliasActions, + newIndexCreation: true, + }); + }); + + it('CREATE_TARGET_INDEX -> INDEX_STATE_UPDATE_DONE when successful and alias actions are empty', () => { + const state = createState(); + const res: StateActionResponse<'CREATE_TARGET_INDEX'> = + Either.right('create_index_succeeded'); + + getAliasActionsMock.mockReturnValue([]); + + const newState = createTargetIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'INDEX_STATE_UPDATE_DONE', + previousMappings: state.indexMappings, + currentIndexMeta: state.indexMappings._meta, + aliases: [], + aliasActions: [], + newIndexCreation: true, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts index bb0697a70cf51..ab0f421822675 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/create_target_index.ts @@ -15,11 +15,10 @@ import { isTypeof } from '../../actions'; import { getAliasActions } from '../../utils'; import type { ModelStage } from '../types'; -export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASES' | 'FATAL'> = ( - state, - res, - context -) => { +export const createTargetIndex: ModelStage< + 'CREATE_TARGET_INDEX', + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' | 'FATAL' +> = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; if (isTypeof(left, 'index_not_green_timeout')) { @@ -48,10 +47,11 @@ export const createTargetIndex: ModelStage<'CREATE_TARGET_INDEX', 'UPDATE_ALIASE return { ...state, - controlState: 'UPDATE_ALIASES', + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', previousMappings: state.indexMappings, currentIndexMeta, aliases: [], aliasActions, + newIndexCreation: true, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts new file mode 100644 index 0000000000000..a16bed57fec65 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.mocks.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const getOutdatedDocumentsQueryMock = jest.fn(); +export const createDocumentTransformFnMock = jest.fn(); + +jest.doMock('../../utils', () => { + const realModule = jest.requireActual('../../utils'); + return { + ...realModule, + getOutdatedDocumentsQuery: getOutdatedDocumentsQueryMock, + createDocumentTransformFn: createDocumentTransformFnMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts new file mode 100644 index 0000000000000..abf8a689ad621 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.test.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + getOutdatedDocumentsQueryMock, + createDocumentTransformFnMock, +} from './documents_update_init.test.mocks'; +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { ResponseType } from '../../next'; +import type { DocumentsUpdateInitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { documentsUpdateInit } from './documents_update_init'; +import { createType } from '../../test_helpers'; + +describe('Stage: documentsUpdateInit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): DocumentsUpdateInitState => ({ + ...createPostInitState(), + controlState: 'DOCUMENTS_UPDATE_INIT', + ...parts, + }); + + beforeEach(() => { + getOutdatedDocumentsQueryMock.mockReset(); + createDocumentTransformFnMock.mockReset(); + + context = createContextMock(); + context.typeRegistry.registerType(createType({ name: 'foo' })); + context.typeRegistry.registerType(createType({ name: 'bar' })); + }); + + it('calls getOutdatedDocumentsQuery with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledTimes(1); + expect(getOutdatedDocumentsQueryMock).toHaveBeenCalledWith({ + types: ['foo', 'bar'].map((type) => context.typeRegistry.getType(type)), + }); + }); + + it('calls createDocumentTransformFn with the correct parameters', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + documentsUpdateInit(state, res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, context); + + expect(createDocumentTransformFnMock).toHaveBeenCalledTimes(1); + expect(createDocumentTransformFnMock).toHaveBeenCalledWith({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + }); + + it('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED when successful', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState.controlState).toEqual('SET_DOC_MIGRATION_STARTED'); + }); + + it('updates the state with the expected properties', () => { + const state = createState(); + const res: ResponseType<'DOCUMENTS_UPDATE_INIT'> = Either.right('noop' as const); + + const transformRawDocs = jest.fn(); + createDocumentTransformFnMock.mockReturnValue(transformRawDocs); + + const outdatedDocumentsQuery = Symbol(); + getOutdatedDocumentsQueryMock.mockReturnValue(outdatedDocumentsQuery); + + const newState = documentsUpdateInit( + state, + res as StateActionResponse<'DOCUMENTS_UPDATE_INIT'>, + context + ); + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED', + transformRawDocs, + outdatedDocumentsQuery, + excludeFromUpgradeFilterHooks: expect.any(Object), + excludeOnUpgradeQuery: expect.any(Object), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts new file mode 100644 index 0000000000000..a4f94e3cbe7b2 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/documents_update_init.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { excludeUnusedTypesQuery } from '../../../core'; +import type { ModelStage } from '../types'; +import { getOutdatedDocumentsQuery, createDocumentTransformFn } from '../../utils'; + +export const documentsUpdateInit: ModelStage< + 'DOCUMENTS_UPDATE_INIT', + 'SET_DOC_MIGRATION_STARTED' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const excludeFilterHooks = Object.fromEntries( + context.types + .map((name) => context.typeRegistry.getType(name)!) + .filter((type) => !!type.excludeOnUpgrade) + .map((type) => [type.name, type.excludeOnUpgrade!]) + ); + + const types = context.types.map((type) => context.typeRegistry.getType(type)!); + const outdatedDocumentsQuery = getOutdatedDocumentsQuery({ types }); + + const transformRawDocs = createDocumentTransformFn({ + serializer: context.serializer, + documentMigrator: context.documentMigrator, + }); + + return { + ...state, + excludeOnUpgradeQuery: excludeUnusedTypesQuery, + excludeFromUpgradeFilterHooks: excludeFilterHooks, + outdatedDocumentsQuery, + transformRawDocs, + controlState: 'SET_DOC_MIGRATION_STARTED', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts index 0322f92eb35aa..532f15b5417ed 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index.ts @@ -12,3 +12,18 @@ export { updateAliases } from './update_aliases'; export { updateIndexMappings } from './update_index_mappings'; export { updateIndexMappingsWaitForTask } from './update_index_mappings_wait_for_task'; export { updateMappingModelVersion } from './update_mapping_model_version'; +export { indexStateUpdateDone } from './index_state_update_done'; +export { documentsUpdateInit } from './documents_update_init'; +export { setDocMigrationStarted } from './set_doc_migration_started'; +export { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; +export { cleanupUnknownAndExcludedDocs } from './cleanup_unknown_and_excluded_docs'; +export { cleanupUnknownAndExcludedDocsWaitForTask } from './cleanup_unknown_and_excluded_docs_wait_for_task'; +export { cleanupUnknownAndExcludedDocsRefresh } from './cleanup_unknown_and_excluded_docs_refresh'; +export { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; +export { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; +export { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; +export { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; +export { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; +export { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; +export { updateDocumentModelVersion } from './update_document_model_version'; +export { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts new file mode 100644 index 0000000000000..2b523887109b8 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { IndexStateUpdateDoneState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { indexStateUpdateDone } from './index_state_update_done'; + +describe('Stage: indexStateUpdateDone', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): IndexStateUpdateDoneState => ({ + ...createPostDocInitState(), + controlState: 'INDEX_STATE_UPDATE_DONE', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT when successful and newIndexCreation is false', () => { + const state = createState({ + newIndexCreation: false, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }); + }); + + it('INDEX_STATE_UPDATE_DONE -> DONE when successful and newIndexCreation is true', () => { + const state = createState({ + newIndexCreation: true, + }); + const res = Either.right('noop') as StateActionResponse<'INDEX_STATE_UPDATE_DONE'>; + + const newState = indexStateUpdateDone(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts new file mode 100644 index 0000000000000..2ca41886c8b6c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/index_state_update_done.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const indexStateUpdateDone: ModelStage< + 'INDEX_STATE_UPDATE_DONE', + 'DOCUMENTS_UPDATE_INIT' | 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + if (state.newIndexCreation) { + // we created the index, so we can safely skip the whole document migration + // and go directly to DONE + return { + ...state, + controlState: 'DONE', + }; + } else { + return { + ...state, + controlState: 'DOCUMENTS_UPDATE_INIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts index 8f773fe951171..89c513fa2a66d 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.mocks.ts @@ -10,6 +10,7 @@ export const getCurrentIndexMock = jest.fn(); export const checkVersionCompatibilityMock = jest.fn(); export const buildIndexMappingsMock = jest.fn(); export const generateAdditiveMappingDiffMock = jest.fn(); +export const getAliasActionsMock = jest.fn(); jest.doMock('../../utils', () => { const realModule = jest.requireActual('../../utils'); @@ -19,5 +20,6 @@ jest.doMock('../../utils', () => { checkVersionCompatibility: checkVersionCompatibilityMock, buildIndexMappings: buildIndexMappingsMock, generateAdditiveMappingDiff: generateAdditiveMappingDiffMock, + getAliasActions: getAliasActionsMock, }; }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts index ea6f4424404ef..d8c176af4be0a 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.test.ts @@ -11,6 +11,7 @@ import { checkVersionCompatibilityMock, buildIndexMappingsMock, generateAdditiveMappingDiffMock, + getAliasActionsMock, } from './init.test.mocks'; import * as Either from 'fp-ts/lib/Either'; import { FetchIndexResponse } from '../../../actions'; @@ -49,6 +50,7 @@ describe('Stage: init', () => { status: 'equal', }); generateAdditiveMappingDiffMock.mockReset().mockReturnValue({}); + getAliasActionsMock.mockReset().mockReturnValue([]); context = createContextMock({ indexPrefix: '.kibana', types: ['foo', 'bar'] }); context.typeRegistry.registerType({ @@ -65,7 +67,7 @@ describe('Stage: init', () => { }); }); - it('loops to INIT when cluster routing allocation is incompatible', () => { + it('INIT -> INIT when cluster routing allocation is incompatible', () => { const state = createState(); const res: StateActionResponse<'INIT'> = Either.left({ type: 'incompatible_cluster_routing_allocation', @@ -124,7 +126,7 @@ describe('Stage: init', () => { }); }); - it('forwards to CREATE_TARGET_INDEX', () => { + it('INIT -> CREATE_TARGET_INDEX', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -164,7 +166,7 @@ describe('Stage: init', () => { }); }); - it('forwards to UPDATE_INDEX_MAPPINGS', () => { + it('INIT -> UPDATE_INDEX_MAPPINGS', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -182,6 +184,7 @@ describe('Stage: init', () => { currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, additiveMappingChanges: { someToken: {} }, + newIndexCreation: false, }) ); }); @@ -203,7 +206,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `equal`', () => { - it('forwards to UPDATE_ALIASES', () => { + it('INIT -> UPDATE_ALIASES if alias actions are not empty', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -211,6 +214,7 @@ describe('Stage: init', () => { checkVersionCompatibilityMock.mockReturnValue({ status: 'equal', }); + getAliasActionsMock.mockReturnValue([{ add: { index: '.kibana_1', alias: '.kibana' } }]); const newState = init(state, res, context); @@ -219,6 +223,29 @@ describe('Stage: init', () => { controlState: 'UPDATE_ALIASES', currentIndex, previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, + }) + ); + }); + + it('INIT -> INDEX_STATE_UPDATE_DONE if alias actions are empty', () => { + const state = createState(); + const fetchIndexResponse = createResponse(); + const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); + + checkVersionCompatibilityMock.mockReturnValue({ + status: 'equal', + }); + getAliasActionsMock.mockReturnValue([]); + + const newState = init(state, res, context); + + expect(newState).toEqual( + expect.objectContaining({ + controlState: 'INDEX_STATE_UPDATE_DONE', + currentIndex, + previousMappings: fetchIndexResponse[currentIndex].mappings, + newIndexCreation: false, }) ); }); @@ -240,7 +267,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `lesser`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); @@ -276,7 +303,7 @@ describe('Stage: init', () => { }); describe('when checkVersionCompatibility returns `conflict`', () => { - it('forwards to FATAL', () => { + it('INIT -> FATAL', () => { const state = createState(); const fetchIndexResponse = createResponse(); const res: StateActionResponse<'INIT'> = Either.right(fetchIndexResponse); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts index da138730364f4..19538cc1a4a85 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/init.ts @@ -23,7 +23,11 @@ import type { ModelStage } from '../types'; export const init: ModelStage< 'INIT', - 'CREATE_TARGET_INDEX' | 'UPDATE_INDEX_MAPPINGS' | 'UPDATE_ALIASES' | 'FATAL' + | 'CREATE_TARGET_INDEX' + | 'UPDATE_INDEX_MAPPINGS' + | 'UPDATE_ALIASES' + | 'INDEX_STATE_UPDATE_DONE' + | 'FATAL' > = (state, res, context) => { if (Either.isLeft(res)) { const left = res.left; @@ -78,6 +82,15 @@ export const init: ModelStage< // cloning as we may be mutating it in later stages. const currentIndexMeta = cloneDeep(currentMappings._meta!); + const commonState = { + logs, + currentIndex, + currentIndexMeta, + aliases, + aliasActions, + previousMappings: currentMappings, + }; + switch (versionCheck.status) { // app version is greater than the index mapping version. // scenario of an upgrade: we need to update the mappings @@ -90,13 +103,9 @@ export const init: ModelStage< return { ...state, controlState: 'UPDATE_INDEX_MAPPINGS', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + ...commonState, additiveMappingChanges, + newIndexCreation: false, }; // app version and index mapping version are the same. // either application upgrade without model change, or a simple reboot on the same version. @@ -104,13 +113,9 @@ export const init: ModelStage< case 'equal': return { ...state, - controlState: 'UPDATE_ALIASES', - logs, - currentIndex, - currentIndexMeta, - aliases, - aliasActions, - previousMappings: currentMappings, + controlState: aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + ...commonState, + newIndexCreation: false, }; // app version is lower than the index mapping version. // likely a rollback scenario - unsupported for the initial implementation diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts new file mode 100644 index 0000000000000..323a8ba646875 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchBulkIndexState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchBulkIndex } from './outdated_documents_search_bulk_index'; + +describe('Stage: outdatedDocumentsSearchBulkIndex', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchBulkIndexState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: [], + currentBatch: 0, + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when there are remaining batches', () => { + const state = createState({ + currentBatch: 0, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res = Either.right( + 'bulk_index_succeeded' + ) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'>; + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 1, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ when there are no remaining batches', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = + Either.right('bulk_index_succeeded'); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> FATAL in case of request_entity_too_large_exception', () => { + const state = createState({ + currentBatch: 1, + bulkOperationBatches: [[{ create: {} }], [{ create: {} }]], + }); + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'> = Either.left({ + type: 'request_entity_too_large_exception', + }); + + const newState = outdatedDocumentsSearchBulkIndex(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts new file mode 100644 index 0000000000000..4d6bba19c5154 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_bulk_index.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { FATAL_REASON_REQUEST_ENTITY_TOO_LARGE } from '../../../common/constants'; +import { throwBadResponse } from '../../../model/helpers'; +import { isTypeof } from '../../actions'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchBulkIndex: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + if (isTypeof(res.left, 'request_entity_too_large_exception')) { + return { + ...state, + controlState: 'FATAL', + reason: FATAL_REASON_REQUEST_ENTITY_TOO_LARGE, + }; + } else if ( + isTypeof(res.left, 'target_index_had_write_block') || + isTypeof(res.left, 'index_not_found_exception') + ) { + // we fail on these errors since the target index will never get + // deleted and should only have a write block if a newer version of + // Kibana started an upgrade + throwBadResponse(state, res.left as never); + } else { + throwBadResponse(state, res.left); + } + } + + if (state.currentBatch + 1 < state.bulkOperationBatches.length) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: state.currentBatch + 1, + }; + } + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: true, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts new file mode 100644 index 0000000000000..a5db9fd119a80 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchClosePitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchClosePit } from './outdated_documents_search_close_pit'; + +describe('Stage: outdatedDocumentsSearchClosePit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchClosePitState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH when documents were transformed', () => { + const state = createState({ + hasTransformedDocs: true, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> UPDATE_DOCUMENT_MODEL_VERSIONS when no documents were transformed', () => { + const state = createState({ + hasTransformedDocs: false, + }); + const res = Either.right({}) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'>; + + const newState = outdatedDocumentsSearchClosePit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts new file mode 100644 index 0000000000000..a467a4a7d90fa --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_close_pit.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchClosePit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH' | 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const { hasTransformedDocs } = state; + if (hasTransformedDocs) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + }; + } else { + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts new file mode 100644 index 0000000000000..90417d66f1a3d --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.test.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchOpenPitState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchOpenPit } from './outdated_documents_search_open_pit'; + +describe('Stage: outdatedDocumentsSearchOpenPit', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchOpenPitState => ({ + ...createPostDocInitState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ when successful', () => { + const state = createState(); + const res = Either.right({ + pitId: '42', + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'>; + + const newState = outdatedDocumentsSearchOpenPit(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: { + processed: undefined, + total: undefined, + }, + hasTransformedDocs: false, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts new file mode 100644 index 0000000000000..f7a76e57779c3 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_open_pit.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { createInitialProgress } from '../../../model/progress'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchOpenPit: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT', + 'OUTDATED_DOCUMENTS_SEARCH_READ' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + const pitId = res.right.pitId; + + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + pitId, + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + progress: createInitialProgress(), + hasTransformedDocs: false, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts new file mode 100644 index 0000000000000..de312c966dbda --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.test.ts @@ -0,0 +1,178 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchReadState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRead } from './outdated_documents_search_read'; + +describe('Stage: outdatedDocumentsSearchRead', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchReadState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM when outdated documents are found', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const outdatedDocuments = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + outdatedDocuments, + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments, + lastHitSortValue: [12, 24], + logs: expect.any(Array), + progress: { + total: 9000, + processed: 0, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when no more outdated documents', () => { + const state = createState({ + progress: { + total: 300, + processed: 0, + }, + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when corrupt ids are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'FATAL', + reason: expect.any(String), + }); + }); + + //// + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when corrupt ids are are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + corruptDocumentIds: ['foo_1', 'bar_2'], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT when transform errors are found and discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: true, + }); + const state = createState({ + transformErrors: [{ rawId: 'foo_1', err: new Error('woups') }], + }); + const res = Either.right({ + outdatedDocuments: [], + lastHitSortValue: [12, 24], + totalHits: 9000, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_READ'>; + + const newState = outdatedDocumentsSearchRead(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + logs: expect.any(Array), + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts new file mode 100644 index 0000000000000..d3e8b2b5b9018 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_read.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { logProgress, setProgressTotal } from '../../../model/progress'; +import { + extractDiscardedCorruptDocs, + extractTransformFailuresReason, +} from '../../../model/extract_errors'; + +export const outdatedDocumentsSearchRead: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_READ', + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' | 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' | 'FATAL' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + let logs = state.logs; + + if (res.right.outdatedDocuments.length > 0) { + // search returned outdated documents, so we process them + const progress = setProgressTotal(state.progress, res.right.totalHits); + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: res.right.outdatedDocuments, + lastHitSortValue: res.right.lastHitSortValue, + logs: logProgress(state.logs, progress), + progress, + }; + } else { + // no more outdated documents , we need to move on + if (state.corruptDocumentIds.length > 0 || state.transformErrors.length > 0) { + if (!context.discardCorruptObjects) { + const transformFailureReason = extractTransformFailuresReason( + context.migrationDocLinks.resolveMigrationFailures, + state.corruptDocumentIds, + state.transformErrors + ); + return { + ...state, + controlState: 'FATAL', + reason: transformFailureReason, + }; + } + + // at this point, users have configured kibana to discard corrupt objects + // thus, we can ignore corrupt documents and transform errors and proceed with the migration + logs = [ + ...state.logs, + { + level: 'warning', + message: extractDiscardedCorruptDocs(state.corruptDocumentIds, state.transformErrors), + }, + ]; + } + + // If there are no more results we have transformed all outdated + // documents and we didn't encounter any corrupt documents or transformation errors + // and can proceed to the next step + return { + ...state, + logs, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT', + }; + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts new file mode 100644 index 0000000000000..9584d97b2ace1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchRefreshState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchRefresh } from './outdated_documents_search_refresh'; + +describe('Stage: outdatedDocumentsSearchRefresh', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchRefreshState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS when successful', () => { + const state = createState({}); + const res = Either.right({ + refreshed: true, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_REFRESH'>; + + const newState = outdatedDocumentsSearchRefresh(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts new file mode 100644 index 0000000000000..0816174c5b585 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_refresh.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const outdatedDocumentsSearchRefresh: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH', + 'UPDATE_DOCUMENT_MODEL_VERSIONS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts new file mode 100644 index 0000000000000..6c08e34678214 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.test.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + createSavedObjectRawDoc, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { OutdatedDocumentsSearchTransformState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { outdatedDocumentsSearchTransform } from './outdated_documents_search_transform'; + +describe('Stage: outdatedDocumentsSearchTransform', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): OutdatedDocumentsSearchTransformState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + outdatedDocuments: [], + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX when outdated documents were converted', () => { + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + const res = Either.right({ + processedDocs, + }) as StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'>; + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + currentBatch: 0, + hasTransformedDocs: true, + bulkOperationBatches: expect.any(Array), + progress: { + processed: 2, + total: 100, + }, + }); + }); + + it('OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ in case of documents_transform_failed when discardCorruptObjects is false', () => { + context = createContextMock({ + discardCorruptObjects: false, + }); + const state = createState({ + progress: { + processed: 0, + total: 100, + }, + outdatedDocuments: [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ], + corruptDocumentIds: ['init_1'], + }); + const processedDocs = [ + createSavedObjectRawDoc({ _id: '3' }), + createSavedObjectRawDoc({ _id: '4' }), + ]; + const res: StateActionResponse<'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'> = Either.left({ + type: 'documents_transform_failed', + processedDocs, + corruptDocumentIds: ['foo_1', 'bar_2'], + transformErrors: [], + }); + + const newState = outdatedDocumentsSearchTransform(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: ['init_1', 'foo_1', 'bar_2'], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: 2, + total: 100, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts new file mode 100644 index 0000000000000..b174a57096e17 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/outdated_documents_search_transform.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; +import { incrementProcessedProgress } from '../../../model/progress'; +import { fatalReasonDocumentExceedsMaxBatchSizeBytes } from '../../../model/extract_errors'; +import { createBatches } from '../../../model/create_batches'; +import { isTypeof } from '../../actions'; + +export const outdatedDocumentsSearchTransform: ModelStage< + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM', + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' | 'OUTDATED_DOCUMENTS_SEARCH_READ' | 'FATAL' +> = (state, res, context) => { + // Increment the processed documents, no matter what the results are. + // Otherwise the progress might look off when there are errors. + const progress = incrementProcessedProgress(state.progress, state.outdatedDocuments.length); + const discardCorruptObjects = context.discardCorruptObjects; + if ( + Either.isRight(res) || + (isTypeof(res.left, 'documents_transform_failed') && discardCorruptObjects) + ) { + // we might have some transformation errors, but user has chosen to discard them + if ( + (state.corruptDocumentIds.length === 0 && state.transformErrors.length === 0) || + discardCorruptObjects + ) { + const documents = Either.isRight(res) ? res.right.processedDocs : res.left.processedDocs; + + let corruptDocumentIds = state.corruptDocumentIds; + let transformErrors = state.transformErrors; + + if (Either.isLeft(res)) { + corruptDocumentIds = [...state.corruptDocumentIds, ...res.left.corruptDocumentIds]; + transformErrors = [...state.transformErrors, ...res.left.transformErrors]; + } + + const batches = createBatches({ + documents, + corruptDocumentIds, + transformErrors, + maxBatchSizeBytes: context.migrationConfig.maxBatchSizeBytes.getValueInBytes(), + }); + if (Either.isRight(batches)) { + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX', + bulkOperationBatches: batches.right, + currentBatch: 0, + hasTransformedDocs: true, + progress, + }; + } else { + return { + ...state, + controlState: 'FATAL', + reason: fatalReasonDocumentExceedsMaxBatchSizeBytes({ + _id: batches.left.documentId, + docSizeBytes: batches.left.docSizeBytes, + maxBatchSizeBytes: batches.left.maxBatchSizeBytes, + }), + }; + } + } else { + // We have seen corrupt documents and/or transformation errors + // skip indexing and go straight to reading and transforming more docs + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + progress, + }; + } + } else { + if (isTypeof(res.left, 'documents_transform_failed')) { + // continue to build up any more transformation errors before failing the migration. + return { + ...state, + controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + corruptDocumentIds: [...state.corruptDocumentIds, ...res.left.corruptDocumentIds], + transformErrors: [...state.transformErrors, ...res.left.transformErrors], + hasTransformedDocs: false, + progress, + }; + } else { + throwBadResponse(state, res as never); + } + } +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts new file mode 100644 index 0000000000000..fca84ab858a2f --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.test.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStarted } from './set_doc_migration_started'; + +describe('Stage: setDocMigrationStarted', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES when successful', () => { + const state = createState(); + const res: StateActionResponse<'SET_DOC_MIGRATION_STARTED'> = Either.right( + 'update_mappings_succeeded' as const + ); + + const newState = setDocMigrationStarted(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: { + ...state.currentIndexMeta, + migrationState: { + ...state.currentIndexMeta.migrationState, + convertingDocuments: true, + }, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts new file mode 100644 index 0000000000000..f1c24dbba3861 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationStarted } from '../../utils'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStarted: ModelStage< + 'SET_DOC_MIGRATION_STARTED', + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts new file mode 100644 index 0000000000000..530430d79586e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { SetDocMigrationStartedWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { setDocMigrationStartedWaitForInstances } from './set_doc_migration_started_wait_for_instances'; + +describe('Stage: setDocMigrationStartedWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): SetDocMigrationStartedWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = setDocMigrationStartedWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts new file mode 100644 index 0000000000000..7053f3149579a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/set_doc_migration_started_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const setDocMigrationStartedWaitForInstances: ModelStage< + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES', + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts index 4fac3d02db044..9849e6b464032 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.test.ts @@ -60,7 +60,7 @@ describe('Stage: updateAliases', () => { }); }); - it('UPDATE_ALIASES -> DONE if successful', () => { + it('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE if successful', () => { const state = createState(); const res: StateActionResponse<'UPDATE_ALIASES'> = Either.right('update_aliases_succeeded'); @@ -68,7 +68,7 @@ describe('Stage: updateAliases', () => { expect(newState).toEqual({ ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts index 4d91eb116871b..5d7f0914d0d52 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_aliases.ts @@ -11,7 +11,7 @@ import { throwBadResponse } from '../../../model/helpers'; import { isTypeof } from '../../actions'; import type { ModelStage } from '../types'; -export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( +export const updateAliases: ModelStage<'UPDATE_ALIASES', 'INDEX_STATE_UPDATE_DONE' | 'FATAL'> = ( state, res, context @@ -41,6 +41,6 @@ export const updateAliases: ModelStage<'UPDATE_ALIASES', 'DONE' | 'FATAL'> = ( return { ...state, - controlState: 'DONE', + controlState: 'INDEX_STATE_UPDATE_DONE', }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts new file mode 100644 index 0000000000000..c3d3fd67422b9 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createOutdatedDocumentSearchState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersion } from './update_document_model_version'; + +describe('Stage: updateDocumentModelVersion', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsState => ({ + ...createOutdatedDocumentSearchState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES when successful', () => { + const state = createState({}); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: expect.any(Object), + }); + }); + + it('updates state.currentIndexMeta when successful', () => { + const state = createState({ + currentIndexMeta: { + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 0, bar: 0 }, + migrationState: { + convertingDocuments: true, + }, + }, + }); + const res = Either.right( + 'update_mappings_succeeded' + ) as StateActionResponse<'UPDATE_DOCUMENT_MODEL_VERSIONS'>; + + const newState = updateDocumentModelVersion(state, res, context); + + expect(newState.currentIndexMeta).toEqual({ + mappingVersions: { foo: 1, bar: 2 }, + docVersions: { foo: 1, bar: 2 }, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts new file mode 100644 index 0000000000000..ea3ea142cbd19 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import { setMetaDocMigrationComplete } from '../../utils'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersion: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS', + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + currentIndexMeta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts new file mode 100644 index 0000000000000..b2ea37919418b --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instance.test.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { + createContextMock, + createPostDocInitState, + type MockedMigratorContext, +} from '../../test_helpers'; +import type { UpdateDocumentModelVersionsWaitForInstancesState } from '../../state'; +import type { StateActionResponse } from '../types'; +import { updateDocumentModelVersionWaitForInstances } from './update_document_model_version_wait_for_instances'; + +describe('Stage: updateDocumentModelVersionWaitForInstances', () => { + let context: MockedMigratorContext; + + const createState = ( + parts: Partial = {} + ): UpdateDocumentModelVersionsWaitForInstancesState => ({ + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + ...parts, + }); + + beforeEach(() => { + context = createContextMock(); + }); + + it('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE when successful', () => { + const state = createState(); + const res = Either.right( + 'wait_succeeded' as const + ) as StateActionResponse<'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'>; + + const newState = updateDocumentModelVersionWaitForInstances(state, res, context); + + expect(newState).toEqual({ + ...state, + controlState: 'DONE', + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts new file mode 100644 index 0000000000000..82e3ce7c10cdd --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_document_model_version_wait_for_instances.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Either from 'fp-ts/lib/Either'; +import { throwBadResponse } from '../../../model/helpers'; +import type { ModelStage } from '../types'; + +export const updateDocumentModelVersionWaitForInstances: ModelStage< + 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES', + 'DONE' +> = (state, res, context) => { + if (Either.isLeft(res)) { + throwBadResponse(state, res as never); + } + + return { + ...state, + controlState: 'DONE', + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts index 9856cb0c5a1e5..c47ef54030b3b 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_index_mappings_wait_for_task.ts @@ -33,11 +33,5 @@ export const updateIndexMappingsWaitForTask: ModelStage< return { ...state, controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', - currentIndexMeta: { - ...state.currentIndexMeta, - mappingVersions: { - ...context.typeModelVersions, - }, - }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts index 971482d3262b7..b21ec69a531f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.test.ts @@ -12,20 +12,19 @@ import { createPostInitState, type MockedMigratorContext, } from '../../test_helpers'; -import type { ResponseType } from '../../next'; -import type { UpdateIndexMappingsState } from '../../state'; +import type { UpdateMappingModelVersionState } from '../../state'; import type { StateActionResponse } from '../types'; -import { updateIndexMappings } from './update_index_mappings'; +import { updateMappingModelVersion } from './update_mapping_model_version'; +import { setMetaMappingMigrationComplete } from '../../utils'; -describe('Stage: updateIndexMappings', () => { +describe('Stage: updateMappingModelVersion', () => { let context: MockedMigratorContext; const createState = ( - parts: Partial = {} - ): UpdateIndexMappingsState => ({ + parts: Partial = {} + ): UpdateMappingModelVersionState => ({ ...createPostInitState(), - controlState: 'UPDATE_INDEX_MAPPINGS', - additiveMappingChanges: {}, + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', ...parts, }); @@ -33,21 +32,50 @@ describe('Stage: updateIndexMappings', () => { context = createContextMock(); }); - it('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK when successful', () => { - const state = createState(); - const res: ResponseType<'UPDATE_INDEX_MAPPINGS'> = Either.right({ - taskId: '42', + it('updates state.currentIndexMeta', () => { + const state = createState({}); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); + + const newState = updateMappingModelVersion(state, res, context); + expect(newState.currentIndexMeta).toEqual( + setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }) + ); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> UPDATE_ALIASES when at least one aliasActions', () => { + const state = createState({ + aliasActions: [{ add: { alias: '.kibana', index: '.kibana_1' } }], }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' + ); - const newState = updateIndexMappings( - state, - res as StateActionResponse<'UPDATE_INDEX_MAPPINGS'>, - context + const newState = updateMappingModelVersion(state, res, context); + expect(newState).toEqual({ + ...state, + currentIndexMeta: expect.any(Object), + controlState: 'UPDATE_ALIASES', + }); + }); + + it('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE when no aliasActions', () => { + const state = createState({ + aliasActions: [], + }); + const res: StateActionResponse<'UPDATE_MAPPING_MODEL_VERSIONS'> = Either.right( + 'update_mappings_succeeded' ); + + const newState = updateMappingModelVersion(state, res, context); expect(newState).toEqual({ ...state, - controlState: 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK', - updateTargetMappingsTaskId: '42', + currentIndexMeta: expect.any(Object), + controlState: 'INDEX_STATE_UPDATE_DONE', }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts index 8b4df56fc83a7..946c4a4ab1ef3 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/model/stages/update_mapping_model_version.ts @@ -9,10 +9,11 @@ import * as Either from 'fp-ts/lib/Either'; import { throwBadResponse } from '../../../model/helpers'; import type { ModelStage } from '../types'; +import { setMetaMappingMigrationComplete } from '../../utils'; export const updateMappingModelVersion: ModelStage< 'UPDATE_MAPPING_MODEL_VERSIONS', - 'DONE' | 'FATAL' + 'UPDATE_ALIASES' | 'INDEX_STATE_UPDATE_DONE' > = (state, res, context) => { if (Either.isLeft(res)) { throwBadResponse(state, res as never); @@ -20,6 +21,10 @@ export const updateMappingModelVersion: ModelStage< return { ...state, - controlState: 'DONE', + controlState: state.aliasActions.length ? 'UPDATE_ALIASES' : 'INDEX_STATE_UPDATE_DONE', + currentIndexMeta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts new file mode 100644 index 0000000000000..fc274bcfd1214 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.mocks.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const setMetaMappingMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationCompleteMock = jest.fn(); +export const setMetaDocMigrationStartedMock = jest.fn(); + +jest.doMock('./utils', () => { + const actual = jest.requireActual('./utils'); + return { + ...actual, + setMetaDocMigrationStarted: setMetaDocMigrationStartedMock, + setMetaMappingMigrationComplete: setMetaMappingMigrationCompleteMock, + setMetaDocMigrationComplete: setMetaDocMigrationCompleteMock, + }; +}); + +const realActions = jest.requireActual('./actions'); + +export const ActionMocks = Object.keys(realActions).reduce((mocks, key) => { + mocks[key] = jest.fn().mockImplementation((state: unknown) => state); + return mocks; +}, {} as Record>); + +jest.doMock('./actions', () => ActionMocks); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts new file mode 100644 index 0000000000000..d9135fff65a3e --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.test.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ActionMocks, + setMetaDocMigrationStartedMock, + setMetaDocMigrationCompleteMock, + setMetaMappingMigrationCompleteMock, +} from './next.test.mocks'; +import { nextActionMap, type ActionMap } from './next'; +import { + createContextMock, + type MockedMigratorContext, + createPostDocInitState, +} from './test_helpers'; +import type { + SetDocMigrationStartedState, + UpdateMappingModelVersionState, + UpdateDocumentModelVersionsState, +} from './state'; + +describe('actions', () => { + let context: MockedMigratorContext; + let actionMap: ActionMap; + + beforeEach(() => { + jest.clearAllMocks(); + + context = createContextMock(); + actionMap = nextActionMap(context); + }); + + describe('SET_DOC_MIGRATION_STARTED', () => { + it('calls setMetaDocMigrationStarted with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + action(state); + + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationStartedMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: SetDocMigrationStartedState = { + ...createPostDocInitState(), + controlState: 'SET_DOC_MIGRATION_STARTED', + }; + const action = actionMap.SET_DOC_MIGRATION_STARTED; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationStartedMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_MAPPING_MODEL_VERSIONS', () => { + it('calls setMetaMappingMigrationComplete with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + action(state); + + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaMappingMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateMappingModelVersionState = { + ...createPostDocInitState(), + controlState: 'UPDATE_MAPPING_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_MAPPING_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaMappingMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); + + describe('UPDATE_DOCUMENT_MODEL_VERSIONS', () => { + it('calls setMetaDocMigrationComplete with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + action(state); + + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledTimes(1); + expect(setMetaDocMigrationCompleteMock).toHaveBeenCalledWith({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }); + }); + + it('calls the updateIndexMeta action with the correct parameters', () => { + const state: UpdateDocumentModelVersionsState = { + ...createPostDocInitState(), + controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS', + }; + const action = actionMap.UPDATE_DOCUMENT_MODEL_VERSIONS; + + const someMeta = { some: 'meta' }; + setMetaDocMigrationCompleteMock.mockReturnValue(someMeta); + + action(state); + + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledTimes(1); + expect(ActionMocks.updateIndexMeta).toHaveBeenCalledWith({ + client: context.elasticsearchClient, + index: state.currentIndex, + meta: someMeta, + }); + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts index a85e9bfde6b56..cb3e1b5b5ad27 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/next.ts @@ -15,10 +15,30 @@ import type { UpdateIndexMappingsWaitForTaskState, UpdateMappingModelVersionState, UpdateAliasesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + DocumentsUpdateInitState, + IndexStateUpdateDoneState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + CleanupUnknownAndExcludedDocsRefreshState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './state'; import type { MigratorContext } from './context'; import * as Actions from './actions'; import { createDelayFn } from '../common/utils'; +import { + setMetaMappingMigrationComplete, + setMetaDocMigrationComplete, + setMetaDocMigrationStarted, +} from './utils'; export type ActionMap = ReturnType; @@ -59,19 +79,108 @@ export const nextActionMap = (context: MigratorContext) => { timeout: '60s', }), UPDATE_MAPPING_MODEL_VERSIONS: (state: UpdateMappingModelVersionState) => - Actions.updateMappings({ + Actions.updateIndexMeta({ client, index: state.currentIndex, - mappings: { - properties: {}, - _meta: state.currentIndexMeta, - }, + meta: setMetaMappingMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), }), UPDATE_ALIASES: (state: UpdateAliasesState) => Actions.updateAliases({ client, aliasActions: state.aliasActions, }), + INDEX_STATE_UPDATE_DONE: (state: IndexStateUpdateDoneState) => () => Actions.noop(), + DOCUMENTS_UPDATE_INIT: (state: DocumentsUpdateInitState) => () => Actions.noop(), + SET_DOC_MIGRATION_STARTED: (state: SetDocMigrationStartedState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationStarted({ + meta: state.currentIndexMeta, + }), + }), + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: ( + state: SetDocMigrationStartedWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: (state: CleanupUnknownAndExcludedDocsState) => + Actions.cleanupUnknownAndExcluded({ + client, + indexName: state.currentIndex, + discardUnknownDocs: true, + excludeOnUpgradeQuery: state.excludeOnUpgradeQuery, + excludeFromUpgradeFilterHooks: state.excludeFromUpgradeFilterHooks, + knownTypes: context.types, + removedTypes: context.deletedTypes, + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: ( + state: CleanupUnknownAndExcludedDocsWaitForTaskState + ) => + Actions.waitForDeleteByQueryTask({ + client, + taskId: state.deleteTaskId, + timeout: '120s', + }), + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: (state: CleanupUnknownAndExcludedDocsRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: (state: OutdatedDocumentsSearchOpenPitState) => + Actions.openPit({ + client, + index: state.currentIndex, + }), + OUTDATED_DOCUMENTS_SEARCH_READ: (state: OutdatedDocumentsSearchReadState) => + Actions.readWithPit({ + client, + pitId: state.pitId, + searchAfter: state.lastHitSortValue, + batchSize: context.migrationConfig.batchSize, + query: state.outdatedDocumentsQuery, + }), + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: (state: OutdatedDocumentsSearchTransformState) => + Actions.transformDocs({ + outdatedDocuments: state.outdatedDocuments, + transformRawDocs: state.transformRawDocs, + }), + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: (state: OutdatedDocumentsSearchBulkIndexState) => + Actions.bulkOverwriteTransformedDocuments({ + client, + index: state.currentIndex, + operations: state.bulkOperationBatches[state.currentBatch], + refresh: false, + }), + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: (state: OutdatedDocumentsSearchClosePitState) => + Actions.closePit({ + client, + pitId: state.pitId, + }), + OUTDATED_DOCUMENTS_SEARCH_REFRESH: (state: OutdatedDocumentsSearchRefreshState) => + Actions.refreshIndex({ + client, + index: state.currentIndex, + }), + UPDATE_DOCUMENT_MODEL_VERSIONS: (state: UpdateDocumentModelVersionsState) => + Actions.updateIndexMeta({ + client, + index: state.currentIndex, + meta: setMetaDocMigrationComplete({ + meta: state.currentIndexMeta, + versions: context.typeModelVersions, + }), + }), + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: ( + state: UpdateDocumentModelVersionsWaitForInstancesState + ) => + Actions.waitForDelay({ + delayInSec: context.migrationConfig.zdt.metaPickupSyncDelaySec, + }), }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts index 0f7d28507bb4a..45d958720a912 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/index.ts @@ -21,5 +21,20 @@ export type { AllControlStates, StateFromActionState, StateFromControlState, + IndexStateUpdateDoneState, + DocumentsUpdateInitState, + SetDocMigrationStartedState, + SetDocMigrationStartedWaitForInstancesState, + CleanupUnknownAndExcludedDocsState, + CleanupUnknownAndExcludedDocsWaitForTaskState, + CleanupUnknownAndExcludedDocsRefreshState, + OutdatedDocumentsSearchOpenPitState, + OutdatedDocumentsSearchReadState, + OutdatedDocumentsSearchTransformState, + OutdatedDocumentsSearchBulkIndexState, + OutdatedDocumentsSearchClosePitState, + OutdatedDocumentsSearchRefreshState, + UpdateDocumentModelVersionsState, + UpdateDocumentModelVersionsWaitForInstancesState, } from './types'; export { createInitialState } from './create_initial_state'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts index d43c6e49dd5e5..123df0455c9bf 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/state/types.ts @@ -6,11 +6,18 @@ * Side Public License, v 1. */ -import type { SavedObjectsMappingProperties } from '@kbn/core-saved-objects-server'; +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { + SavedObjectsRawDoc, + SavedObjectsMappingProperties, + SavedObjectTypeExcludeFromUpgradeFilterHook, +} from '@kbn/core-saved-objects-server'; import type { IndexMapping, IndexMappingMeta } from '@kbn/core-saved-objects-base-server-internal'; -import type { MigrationLog } from '../../types'; +import type { MigrationLog, Progress, TransformRawDocs } from '../../types'; import type { ControlState } from '../../state_action_machine'; +import type { BulkOperationBatch } from '../../model/create_batches'; import type { AliasAction } from '../../actions'; +import { TransformErrorObjects } from '../../core'; export interface BaseState extends ControlState { readonly retryCount: number; @@ -23,6 +30,9 @@ export interface InitState extends BaseState { readonly controlState: 'INIT'; } +/** + * Common state properties available after the `INIT` stage + */ export interface PostInitState extends BaseState { /** * The index we're currently migrating. @@ -46,6 +56,35 @@ export interface PostInitState extends BaseState { * All operations updating this field will update in the state accordingly. */ readonly currentIndexMeta: IndexMappingMeta; + /** + * When true, will fully skip document migration after the INDEX_STATE_UPDATE_DONE stage. + * Used when 'upgrading' a fresh cluster (via CREATE_TARGET_INDEX), as we create + * the index with the correct meta and because we're sure we don't need to migrate documents + * in that case. + */ + readonly newIndexCreation: boolean; +} + +/** + * Common state properties available after the `DOCUMENTS_UPDATE_INIT` stage + */ +export interface PostDocInitState extends PostInitState { + readonly excludeOnUpgradeQuery: QueryDslQueryContainer; + readonly excludeFromUpgradeFilterHooks: Record< + string, + SavedObjectTypeExcludeFromUpgradeFilterHook + >; + readonly outdatedDocumentsQuery: QueryDslQueryContainer; + readonly transformRawDocs: TransformRawDocs; +} + +export interface OutdatedDocumentsSearchState extends PostDocInitState { + readonly pitId: string; + readonly lastHitSortValue: number[] | undefined; + readonly corruptDocumentIds: string[]; + readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; + readonly hasTransformedDocs: boolean; } export interface CreateTargetIndexState extends BaseState { @@ -72,6 +111,72 @@ export interface UpdateAliasesState extends PostInitState { readonly controlState: 'UPDATE_ALIASES'; } +export interface IndexStateUpdateDoneState extends PostInitState { + readonly controlState: 'INDEX_STATE_UPDATE_DONE'; +} + +export interface DocumentsUpdateInitState extends PostInitState { + readonly controlState: 'DOCUMENTS_UPDATE_INIT'; +} + +export interface SetDocMigrationStartedState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED'; +} + +export interface SetDocMigrationStartedWaitForInstancesState extends PostDocInitState { + readonly controlState: 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES'; +} + +export interface CleanupUnknownAndExcludedDocsState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS'; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsWaitForTaskState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK'; + readonly deleteTaskId: string; + readonly hasDeletedDocs?: boolean; +} + +export interface CleanupUnknownAndExcludedDocsRefreshState extends PostDocInitState { + readonly controlState: 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH'; +} + +export interface OutdatedDocumentsSearchOpenPitState extends PostDocInitState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT'; +} + +export interface OutdatedDocumentsSearchReadState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ'; +} + +export interface OutdatedDocumentsSearchTransformState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM'; + readonly outdatedDocuments: SavedObjectsRawDoc[]; +} + +export interface OutdatedDocumentsSearchBulkIndexState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX'; + readonly bulkOperationBatches: BulkOperationBatch[]; + readonly currentBatch: number; +} + +export interface OutdatedDocumentsSearchClosePitState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'; +} + +export interface OutdatedDocumentsSearchRefreshState extends OutdatedDocumentsSearchState { + readonly controlState: 'OUTDATED_DOCUMENTS_SEARCH_REFRESH'; +} + +export interface UpdateDocumentModelVersionsState extends PostDocInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS'; +} + +export interface UpdateDocumentModelVersionsWaitForInstancesState extends PostInitState { + readonly controlState: 'UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES'; +} + /** Migration completed successfully */ export interface DoneState extends BaseState { readonly controlState: 'DONE'; @@ -92,7 +197,22 @@ export type State = | UpdateIndexMappingsState | UpdateIndexMappingsWaitForTaskState | UpdateMappingModelVersionState - | UpdateAliasesState; + | UpdateAliasesState + | IndexStateUpdateDoneState + | DocumentsUpdateInitState + | SetDocMigrationStartedState + | SetDocMigrationStartedWaitForInstancesState + | CleanupUnknownAndExcludedDocsState + | CleanupUnknownAndExcludedDocsWaitForTaskState + | CleanupUnknownAndExcludedDocsRefreshState + | OutdatedDocumentsSearchOpenPitState + | OutdatedDocumentsSearchReadState + | OutdatedDocumentsSearchTransformState + | OutdatedDocumentsSearchBulkIndexState + | OutdatedDocumentsSearchClosePitState + | UpdateDocumentModelVersionsState + | UpdateDocumentModelVersionsWaitForInstancesState + | OutdatedDocumentsSearchRefreshState; export type AllControlStates = State['controlState']; @@ -110,6 +230,21 @@ export interface ControlStateMap { UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK: UpdateIndexMappingsWaitForTaskState; UPDATE_MAPPING_MODEL_VERSIONS: UpdateMappingModelVersionState; UPDATE_ALIASES: UpdateAliasesState; + INDEX_STATE_UPDATE_DONE: IndexStateUpdateDoneState; + DOCUMENTS_UPDATE_INIT: DocumentsUpdateInitState; + SET_DOC_MIGRATION_STARTED: SetDocMigrationStartedState; + SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES: SetDocMigrationStartedWaitForInstancesState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS: CleanupUnknownAndExcludedDocsState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK: CleanupUnknownAndExcludedDocsWaitForTaskState; + CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_REFRESH: CleanupUnknownAndExcludedDocsRefreshState; + OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT: OutdatedDocumentsSearchOpenPitState; + OUTDATED_DOCUMENTS_SEARCH_READ: OutdatedDocumentsSearchReadState; + OUTDATED_DOCUMENTS_SEARCH_TRANSFORM: OutdatedDocumentsSearchTransformState; + OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX: OutdatedDocumentsSearchBulkIndexState; + OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT: OutdatedDocumentsSearchClosePitState; + OUTDATED_DOCUMENTS_SEARCH_REFRESH: OutdatedDocumentsSearchRefreshState; + UPDATE_DOCUMENT_MODEL_VERSIONS: UpdateDocumentModelVersionsState; + UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES: UpdateDocumentModelVersionsWaitForInstancesState; } /** diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts index faf9f9c89c9f5..0dafc36108b93 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/context.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import { ByteSizeValue } from '@kbn/config-schema'; import { ElasticsearchClientMock, elasticsearchClientMock, @@ -14,6 +15,7 @@ import { SavedObjectTypeRegistry } from '@kbn/core-saved-objects-base-server-int import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; import { docLinksServiceMock } from '@kbn/core-doc-links-server-mocks'; import type { MigratorContext } from '../context'; +import { createDocumentMigrator } from './document_migrator'; export type MockedMigratorContext = Omit & { elasticsearchClient: ElasticsearchClientMock; @@ -33,12 +35,26 @@ export const createContextMock = ( foo: 1, bar: 2, }, + documentMigrator: createDocumentMigrator(), + migrationConfig: { + algorithm: 'zdt', + batchSize: 1000, + maxBatchSizeBytes: new ByteSizeValue(1e8), + pollInterval: 0, + scrollDuration: '0s', + skip: false, + retryAttempts: 5, + zdt: { + metaPickupSyncDelaySec: 120, + }, + }, elasticsearchClient: elasticsearchClientMock.createElasticsearchClient(), maxRetryAttempts: 15, migrationDocLinks: docLinksServiceMock.createSetupContract().links.kibanaUpgradeSavedObjects, typeRegistry, serializer: serializerMock.create(), deletedTypes: ['deleted-type'], + discardCorruptObjects: false, ...parts, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts new file mode 100644 index 0000000000000..524da05952055 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/document_migrator.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { VersionedTransformer } from '../../document_migrator'; + +export const createDocumentMigrator = (): jest.Mocked => { + return { + migrationVersion: {}, + migrate: jest.fn().mockImplementation((doc: unknown) => doc), + migrateAndConvert: jest.fn().mockImplementation((doc: unknown) => [doc]), + prepareMigrations: jest.fn(), + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts index 5658828fc2e0c..8b79eef0069f0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/index.ts @@ -7,5 +7,11 @@ */ export { createContextMock, type MockedMigratorContext } from './context'; -export { createPostInitState } from './state'; +export { + createPostInitState, + createPostDocInitState, + createOutdatedDocumentSearchState, +} from './state'; export { createType } from './saved_object_type'; +export { createDocumentMigrator } from './document_migrator'; +export { createSavedObjectRawDoc } from './saved_object'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts new file mode 100644 index 0000000000000..9b5520a8508db --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/saved_object.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; + +export const createSavedObjectRawDoc = ( + parts: Partial +): SavedObjectsRawDoc => ({ + _id: '42', + _source: { + type: 'some-type', + }, + ...parts, +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts index bd95881abbba4..b91f2482326cd 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/test_helpers/state.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { PostInitState } from '../state/types'; +import { PostInitState, PostDocInitState, OutdatedDocumentsSearchState } from '../state/types'; export const createPostInitState = (): PostInitState => ({ controlState: 'INIT', @@ -18,4 +18,26 @@ export const createPostInitState = (): PostInitState => ({ aliasActions: [], previousMappings: { properties: {} }, currentIndexMeta: {}, + newIndexCreation: false, +}); + +export const createPostDocInitState = (): PostDocInitState => ({ + ...createPostInitState(), + excludeOnUpgradeQuery: { bool: {} }, + excludeFromUpgradeFilterHooks: {}, + outdatedDocumentsQuery: { bool: {} }, + transformRawDocs: jest.fn(), +}); + +export const createOutdatedDocumentSearchState = (): OutdatedDocumentsSearchState => ({ + ...createPostDocInitState(), + pitId: '42', + lastHitSortValue: undefined, + corruptDocumentIds: [], + transformErrors: [], + hasTransformedDocs: false, + progress: { + processed: undefined, + total: undefined, + }, }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts index f4536cf1c75b0..35354001f6803 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.test.ts @@ -78,6 +78,9 @@ describe('buildIndexMeta', () => { bar: 1, dolly: 3, }, + migrationState: { + convertingDocuments: false, + }, }); }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts index 6221221ab993c..a75ebd4dbdc1e 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/build_index_mappings.ts @@ -53,5 +53,8 @@ export const buildIndexMeta = ({ types }: BuildIndexMetaOpts): IndexMappingMeta return { mappingVersions: modelVersions, docVersions: modelVersions, + migrationState: { + convertingDocuments: false, + }, }; }; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts index 6ad12656229fc..8430f1f898426 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.test.ts @@ -62,6 +62,7 @@ describe('checkVersionCompatibility', () => { expect(getModelVersionsFromMappingsMock).toHaveBeenCalledWith({ mappings, source: 'mappingVersions', + knownTypes: ['foo', 'bar'], }); }); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts index 4499ce419d34a..c231645fb7993 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/check_version_compatibility.ts @@ -29,7 +29,11 @@ export const checkVersionCompatibility = ({ deletedTypes, }: CheckVersionCompatibilityOpts): CompareModelVersionResult => { const appVersions = getModelVersionMapForTypes(types); - const indexVersions = getModelVersionsFromMappings({ mappings, source }); + const indexVersions = getModelVersionsFromMappings({ + mappings, + source, + knownTypes: types.map((type) => type.name), + }); if (!indexVersions) { throw new Error(`Cannot check version: ${source} not present in the mapping meta`); } diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts index f23b1e84a87ea..400e01e999797 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/generate_additive_mapping_diff.ts @@ -36,7 +36,11 @@ export const generateAdditiveMappingDiff = ({ deletedTypes, }: GenerateAdditiveMappingsDiffOpts): SavedObjectsMappingProperties => { const typeVersions = getModelVersionMapForTypes(types); - const mappingVersion = getModelVersionsFromMappingMeta({ meta, source: 'mappingVersions' }); + const mappingVersion = getModelVersionsFromMappingMeta({ + meta, + source: 'mappingVersions', + knownTypes: types.map((type) => type.name), + }); if (!mappingVersion) { // should never occur given we checked previously in the flow but better safe than sorry. throw new Error( diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts index ebc22e623f600..76c66a9fc9bd0 100644 --- a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/index.ts @@ -12,3 +12,10 @@ export { checkVersionCompatibility } from './check_version_compatibility'; export { buildIndexMappings, buildIndexMeta } from './build_index_mappings'; export { getAliasActions } from './get_alias_actions'; export { generateAdditiveMappingDiff } from './generate_additive_mapping_diff'; +export { getOutdatedDocumentsQuery } from './outdated_documents_query'; +export { createDocumentTransformFn } from './transform_raw_docs'; +export { + setMetaMappingMigrationComplete, + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, +} from './update_index_meta'; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts new file mode 100644 index 0000000000000..f39016b7d86a1 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { getOutdatedDocumentsQuery } from './outdated_documents_query'; +import { createType } from '../test_helpers/saved_object_type'; + +const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +describe('getOutdatedDocumentsQuery', () => { + it('generates the correct query', () => { + const fooType = createType({ + name: 'foo', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + }, + }); + const barType = createType({ + name: 'bar', + modelVersions: { + 1: dummyModelVersion, + 2: dummyModelVersion, + 3: dummyModelVersion, + }, + }); + + const query = getOutdatedDocumentsQuery({ + types: [fooType, barType], + }); + + expect(query).toMatchInlineSnapshot(` + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "foo", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.foo": "10.2.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.2.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + Object { + "bool": Object { + "must": Array [ + Object { + "term": Object { + "type": "bar", + }, + }, + Object { + "bool": Object { + "should": Array [ + Object { + "bool": Object { + "must": Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + "must_not": Object { + "term": Object { + "migrationVersion.bar": "10.3.0", + }, + }, + }, + }, + Object { + "bool": Object { + "must_not": Array [ + Object { + "exists": Object { + "field": "migrationVersion", + }, + }, + Object { + "term": Object { + "typeMigrationVersion": "10.3.0", + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + }, + ], + }, + } + `); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts new file mode 100644 index 0000000000000..e15a2e447b554 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/outdated_documents_query.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types'; +import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import { + getModelVersionMapForTypes, + modelVersionToVirtualVersion, +} from '@kbn/core-saved-objects-base-server-internal'; + +interface GetOutdatedDocumentsQueryOps { + types: SavedObjectsType[]; +} + +export const getOutdatedDocumentsQuery = ({ + types, +}: GetOutdatedDocumentsQueryOps): QueryDslQueryContainer => { + // Note: in theory, we could check the difference of model version with the index's + // and narrow the search filter only on the type that have different versions. + // however, it feels safer to just search for all outdated document, just in case. + const modelVersions = getModelVersionMapForTypes(types); + return { + bool: { + should: types.map((type) => { + const virtualVersion = modelVersionToVirtualVersion(modelVersions[type.name]); + return { + bool: { + must: [ + { term: { type: type.name } }, + { + bool: { + should: [ + { + bool: { + must: { exists: { field: 'migrationVersion' } }, + must_not: { term: { [`migrationVersion.${type.name}`]: virtualVersion } }, + }, + }, + { + bool: { + must_not: [ + { exists: { field: 'migrationVersion' } }, + { term: { typeMigrationVersion: virtualVersion } }, + ], + }, + }, + ], + }, + }, + ], + }, + }; + }), + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts new file mode 100644 index 0000000000000..ab6e3f0c5773c --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.mocks.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export const migrateRawDocsSafelyMock = jest.fn(); + +jest.doMock('../../core/migrate_raw_docs', () => { + const actual = jest.requireActual('../../core/migrate_raw_docs'); + return { + ...actual, + migrateRawDocsSafely: migrateRawDocsSafelyMock, + }; +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts new file mode 100644 index 0000000000000..91a1f9983f330 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { migrateRawDocsSafelyMock } from './transform_raw_docs.test.mocks'; +import { serializerMock } from '@kbn/core-saved-objects-base-server-mocks'; +import { createDocumentMigrator, createSavedObjectRawDoc } from '../test_helpers'; +import { createDocumentTransformFn } from './transform_raw_docs'; + +describe('createDocumentTransformFn', () => { + let serializer: ReturnType; + let documentMigrator: ReturnType; + + beforeEach(() => { + migrateRawDocsSafelyMock.mockReset(); + serializer = serializerMock.create(); + documentMigrator = createDocumentMigrator(); + }); + + it('returns a function calling migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + expect(migrateRawDocsSafelyMock).not.toHaveBeenCalled(); + + const documents = [ + createSavedObjectRawDoc({ _id: '1' }), + createSavedObjectRawDoc({ _id: '2' }), + ]; + transformFn(documents); + + expect(migrateRawDocsSafelyMock).toHaveBeenCalledTimes(1); + expect(migrateRawDocsSafelyMock).toHaveBeenCalledWith({ + rawDocs: documents, + serializer, + migrateDoc: documentMigrator.migrateAndConvert, + }); + }); + + it('forward the return from migrateRawDocsSafely', () => { + const transformFn = createDocumentTransformFn({ + serializer, + documentMigrator, + }); + + const documents = [createSavedObjectRawDoc({ _id: '1' })]; + + const expected = Symbol(); + migrateRawDocsSafelyMock.mockReturnValue(expected); + + const result = transformFn(documents); + + expect(result).toBe(expected); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts new file mode 100644 index 0000000000000..1af1cee9d3b84 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/transform_raw_docs.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ISavedObjectsSerializer, SavedObjectsRawDoc } from '@kbn/core-saved-objects-server'; +import { VersionedTransformer } from '../../document_migrator'; +import { TransformRawDocs } from '../../types'; +import { migrateRawDocsSafely } from '../../core/migrate_raw_docs'; + +export interface CreateDocumentTransformFnOpts { + serializer: ISavedObjectsSerializer; + documentMigrator: VersionedTransformer; +} + +export const createDocumentTransformFn = ({ + documentMigrator, + serializer, +}: CreateDocumentTransformFnOpts): TransformRawDocs => { + return (documents: SavedObjectsRawDoc[]) => + migrateRawDocsSafely({ + rawDocs: documents, + migrateDoc: documentMigrator.migrateAndConvert, + serializer, + }); +}; diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts new file mode 100644 index 0000000000000..4298c6b027072 --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; +import { + setMetaDocMigrationStarted, + setMetaDocMigrationComplete, + setMetaMappingMigrationComplete, +} from './update_index_meta'; + +const getDefaultMeta = (): IndexMappingMeta => ({ + mappingVersions: { + foo: 1, + bar: 1, + }, + docVersions: { + foo: 1, + bar: 1, + }, + migrationState: { + convertingDocuments: false, + }, +}); + +describe('setMetaMappingMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaMappingMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + mappingVersions: versions, + }); + }); +}); + +describe('setMetaDocMigrationStarted', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = getDefaultMeta(); + + const updated = setMetaDocMigrationStarted({ meta }); + + expect(updated).toEqual({ + ...meta, + migrationState: { + convertingDocuments: true, + }, + }); + }); +}); + +describe('setMetaDocMigrationComplete', () => { + it('updates the meta to set the mappingVersions', () => { + const meta: IndexMappingMeta = { + ...getDefaultMeta(), + migrationState: { + convertingDocuments: true, + }, + }; + const versions: ModelVersionMap = { foo: 3, bar: 2 }; + + const updated = setMetaDocMigrationComplete({ meta, versions }); + + expect(updated).toEqual({ + ...meta, + docVersions: versions, + migrationState: { + convertingDocuments: false, + }, + }); + }); +}); diff --git a/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts new file mode 100644 index 0000000000000..73d693d4a1f1a --- /dev/null +++ b/packages/core/saved-objects/core-saved-objects-migration-server-internal/src/zdt/utils/update_index_meta.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndexMappingMeta, + ModelVersionMap, +} from '@kbn/core-saved-objects-base-server-internal'; + +export const setMetaMappingMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + mappingVersions: { + ...versions, + }, + }; +}; + +export const setMetaDocMigrationStarted = ({ + meta, +}: { + meta: IndexMappingMeta; +}): IndexMappingMeta => { + return { + ...meta, + migrationState: { + convertingDocuments: true, + }, + }; +}; + +export const setMetaDocMigrationComplete = ({ + meta, + versions, +}: { + meta: IndexMappingMeta; + versions: ModelVersionMap; +}): IndexMappingMeta => { + return { + ...meta, + docVersions: { + ...versions, + }, + migrationState: { + convertingDocuments: false, + }, + }; +}; diff --git a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts index 3094a1ccb3e09..44e8c348778ec 100644 --- a/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts +++ b/packages/core/saved-objects/core-saved-objects-server/src/model_version/model_version.ts @@ -37,10 +37,7 @@ export interface SavedObjectsModelVersion { * * @public */ -export interface SavedObjectsModelExpansionChange< - PreviousAttributes = unknown, - NewAttributes = unknown -> { +export interface SavedObjectsModelExpansionChange { /** * The type of {@link SavedObjectsModelChange | change}, used to identify them internally. */ diff --git a/packages/kbn-axe-config/index.ts b/packages/kbn-axe-config/index.ts index df175dbf08173..7f73408733720 100644 --- a/packages/kbn-axe-config/index.ts +++ b/packages/kbn-axe-config/index.ts @@ -54,5 +54,8 @@ export const AXE_OPTIONS = { bypass: { enabled: false, // disabled because it's too flaky }, + 'nested-interactive': { + enabled: false, // tracker here - https://github.com/elastic/kibana/issues/152494 disabled because we have too many failures on interactive controls + }, }, }; diff --git a/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts new file mode 100644 index 0000000000000..2065dcf27576d --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/jest_matchers.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MatcherFunction } from 'expect'; +import { LogRecord } from '@kbn/logging'; + +const toContainLogEntry: MatcherFunction<[entry: string]> = (actual, entry) => { + if (!Array.isArray(actual)) { + throw new Error('actual must be an array'); + } + const logEntries = actual as LogRecord[]; + if (logEntries.find((item) => item.message.includes(entry))) { + return { + pass: true, + message: () => `Entry "${entry}" found in log file`, + }; + } else { + return { + pass: false, + message: () => `Entry "${entry}" not found in log file`, + }; + } +}; + +expect.extend({ + toContainLogEntry, +}); + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace jest { + interface Matchers { + toContainLogEntry(entry: string): R; + } + } +} diff --git a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts index 0a84b9bc4b7e8..610981bab56ab 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/test_utils.ts @@ -8,9 +8,12 @@ import { Env } from '@kbn/config'; import { getDocLinksMeta, getDocLinks } from '@kbn/doc-links'; +import { LogRecord } from '@kbn/logging'; import { REPO_ROOT } from '@kbn/repo-info'; import { getEnvOptions } from '@kbn/config-mocks'; import type { SavedObjectsType } from '@kbn/core-saved-objects-server'; +import fs from 'fs/promises'; +import JSON5 from 'json5'; export const getDocVersion = () => { const env = Env.createDefault(REPO_ROOT, getEnvOptions()); @@ -33,3 +36,11 @@ export const createType = (parts: Partial): SavedObjectsType = mappings: { properties: {} }, ...parts, }); + +export const parseLogFile = async (filePath: string): Promise => { + const logFileContent = await fs.readFile(filePath, 'utf-8'); + return logFileContent + .split('\n') + .filter(Boolean) + .map((str) => JSON5.parse(str)) as LogRecord[]; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts new file mode 100644 index 0000000000000..3cf5e499eaa6e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base.fixtures.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; +import { createType } from '../test_utils'; +import { type KibanaMigratorTestKitParams } from '../kibana_migrator_test_kit'; + +export const getBaseMigratorParams = (): KibanaMigratorTestKitParams => ({ + kibanaIndex: '.kibana', + kibanaVersion: '8.8.0', + settings: { + migrations: { + algorithm: 'zdt', + zdt: { + metaPickupSyncDelaySec: 5, + }, + }, + }, +}); + +export const dummyModelVersion: SavedObjectsModelVersion = { + modelChange: { + type: 'expansion', + }, +}; + +export const getFooType = () => { + return createType({ + name: 'foo', + mappings: { + properties: { + someField: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + '2': dummyModelVersion, + }, + }); +}; + +export const getBarType = () => { + return createType({ + name: 'bar', + mappings: { + properties: { + aKeyword: { type: 'keyword' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleAType = () => { + return createType({ + name: 'sample_a', + mappings: { + properties: { + keyword: { type: 'keyword' }, + boolean: { type: 'boolean' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getSampleBType = () => { + return createType({ + name: 'sample_b', + mappings: { + properties: { + text: { type: 'text' }, + text2: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getDeletedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'server', + mappings: { + properties: { + text: { type: 'text' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + }); +}; + +export const getExcludedType = () => { + return createType({ + // we cant' easily introduce a deleted type, so we're using an existing one + name: 'excluded', + mappings: { + properties: { + value: { type: 'integer' }, + }, + }, + switchToModelVersionAt: '8.7.0', + modelVersions: { + '1': dummyModelVersion, + }, + excludeOnUpgrade: () => { + return { + bool: { + must: [{ term: { type: 'excluded' } }, { range: { 'excluded.value': { lte: 1 } } }], + }, + }; + }, + }); +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts deleted file mode 100644 index 9bfac3ac5fd49..0000000000000 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/base_types.fixtures.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import { SavedObjectsModelVersion } from '@kbn/core-saved-objects-server'; -import { createType } from '../test_utils'; - -export const dummyModelVersion: SavedObjectsModelVersion = { - modelChange: { - type: 'expansion', - }, -}; - -export const getFooType = () => { - return createType({ - name: 'foo', - mappings: { - properties: { - someField: { type: 'text' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - '2': dummyModelVersion, - }, - }); -}; - -export const getBarType = () => { - return createType({ - name: 'bar', - mappings: { - properties: { - aKeyword: { type: 'keyword' }, - }, - }, - switchToModelVersionAt: '8.7.0', - modelVersions: { - '1': dummyModelVersion, - }, - }); -}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts new file mode 100644 index 0000000000000..418adb0a7894e --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/basic_document_migration.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range, sortBy } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'basic_document_migration.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('migrates the documents', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> we add a new field and bump the model version by one with a migration + + typeA.mappings.properties = { + ...typeA.mappings.properties, + someAddedField: { type: 'keyword' }, + }; + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + someAddedField: `${doc.attributes.keyword}-mig`, + }, + }, + }; + }, + down: jest.fn(), + }, + addedMappings: { + someAddedField: { type: 'keyword' }, + }, + }, + }, + }; + + // typeB -> we add two new model version with migrations + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig2`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + '3': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + return { + document: { + ...doc, + attributes: { + ...doc.attributes, + text2: `${doc.attributes.text2} - mig3`, + }, + }, + }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const indices = await client.indices.get({ index: '.kibana*' }); + expect(Object.keys(indices)).toEqual(['.kibana_1']); + + const index = indices['.kibana_1']; + const mappings = index.mappings ?? {}; + const mappingMeta = mappings._meta ?? {}; + + expect(mappings.properties).toEqual( + expect.objectContaining({ + sample_a: typeA.mappings, + sample_b: typeB.mappings, + }) + ); + + expect(mappingMeta.docVersions).toEqual({ + sample_a: 2, + sample_b: 3, + }); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ type: 'sample_a' }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ type: 'sample_b' }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + + const sampleAData = sortBy(sampleADocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleAData).toEqual([ + { + id: 'a-0', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_0', someAddedField: 'a_0-mig' }, + }, + { + id: 'a-1', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_1', someAddedField: 'a_1-mig' }, + }, + { + id: 'a-2', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_2', someAddedField: 'a_2-mig' }, + }, + { + id: 'a-3', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_3', someAddedField: 'a_3-mig' }, + }, + { + id: 'a-4', + type: 'sample_a', + attributes: { boolean: true, keyword: 'a_4', someAddedField: 'a_4-mig' }, + }, + ]); + + const sampleBData = sortBy(sampleBDocs, 'id').map((object) => ({ + id: object.id, + type: object.type, + attributes: object.attributes, + })); + + expect(sampleBData).toEqual([ + { + id: 'b-0', + type: 'sample_b', + attributes: { text: 'i am number 0', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-1', + type: 'sample_b', + attributes: { text: 'i am number 1', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-2', + type: 'sample_b', + attributes: { text: 'i am number 2', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-3', + type: 'sample_b', + attributes: { text: 'i am number 3', text2: 'some static text - mig2 - mig3' }, + }, + { + id: 'b-4', + type: 'sample_b', + attributes: { text: 'i am number 4', text2: 'some static text - mig2 - mig3' }, + }, + ]); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('Starting to process 10 documents'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts new file mode 100644 index 0000000000000..4f1b0a4bfe468 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/conversion_failures.test.ts @@ -0,0 +1,198 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getSampleAType, getSampleBType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'conversion_failures.test.log'); + +describe('ZDT upgrades - encountering conversion failures', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + beforeEach(async () => { + await fs.unlink(logFilePath).catch(() => {}); + }); + + describe('when discardCorruptObjects is true', () => { + it('completes the migration and discard the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: true, + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('-> DONE'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(0); + expect(sampleBDocs.map((doc) => doc.id).sort()).toEqual(['b-1', 'b-2', 'b-3', 'b-4']); + }); + }); + + describe('when discardCorruptObjects is false', () => { + it('fails the migration with an explicit message and keep the documents', async () => { + const { runMigrations, savedObjectsRepository } = await prepareScenario({ + discardCorruptObjects: false, + }); + + try { + await runMigrations(); + fail('migration should have failed'); + } catch (err) { + const errorMessage = err.message; + expect(errorMessage).toMatch('6 transformation errors were encountered'); + expect(errorMessage).toMatch('error from a-0'); + expect(errorMessage).toMatch('error from a-1'); + expect(errorMessage).toMatch('error from a-2'); + expect(errorMessage).toMatch('error from a-3'); + expect(errorMessage).toMatch('error from a-4'); + expect(errorMessage).toMatch('error from b-0'); + } + + const records = await parseLogFile(logFilePath); + expect(records).toContainLogEntry('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL'); + + const { saved_objects: sampleADocs } = await savedObjectsRepository.find({ + type: 'sample_a', + }); + const { saved_objects: sampleBDocs } = await savedObjectsRepository.find({ + type: 'sample_b', + }); + + expect(sampleADocs).toHaveLength(5); + expect(sampleBDocs).toHaveLength(5); + }); + }); + + const prepareScenario = async ({ discardCorruptObjects }: { discardCorruptObjects: boolean }) => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + // typeA -> migration failing all the documents + typeA.modelVersions = { + ...typeA.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + throw new Error(`error from ${doc.id}`); + }, + down: jest.fn(), + }, + }, + }, + }; + + // typeB -> migration failing the first doc + typeB.modelVersions = { + ...typeB.modelVersions, + '2': { + modelChange: { + type: 'expansion', + transformation: { + up: (doc) => { + if (doc.id === 'b-0') { + throw new Error(`error from ${doc.id}`); + } + return { document: doc }; + }, + down: jest.fn(), + }, + }, + }, + }; + + const baseParams = getBaseMigratorParams(); + if (discardCorruptObjects) { + baseParams!.settings!.migrations!.discardCorruptObjects = '8.7.0'; + } + + const { runMigrations, client, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...baseParams, + logFilePath, + types: [typeA, typeB], + }); + + return { runMigrations, client, savedObjectsRepository }; + }; + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + + try { + await client.indices.delete({ index: '.kibana_1' }); + } catch (e) { + /* index wasn't created, that's fine */ + } + + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { + keyword: `a_${number}`, + boolean: true, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { + text: `i am number ${number}`, + text2: `some static text`, + }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts index e3f45bb561cf6..760b0f113a8ff 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/create_index.test.ts @@ -8,12 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; +import '../jest_matchers'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType } from './base_types.fixtures'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'create_index.test.log'); @@ -47,15 +46,9 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const barType = getBarType(); const { runMigrations, client } = await getKibanaMigratorTestKit({ - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], - settings: { - migrations: { - algorithm: 'zdt', - }, - }, }); const result = await runMigrations(); @@ -77,7 +70,7 @@ describe('ZDT upgrades - running on a fresh cluster', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -95,21 +88,17 @@ describe('ZDT upgrades - running on a fresh cluster', () => { foo: 2, bar: 1, }, + migrationState: expect.objectContaining({ + convertingDocuments: false, + }), }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - expectLogsContains('INIT -> CREATE_TARGET_INDEX'); - expectLogsContains('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); - expectLogsContains('UPDATE_ALIASES -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> CREATE_TARGET_INDEX'); + expect(records).toContainLogEntry('CREATE_TARGET_INDEX -> UPDATE_ALIASES'); + expect(records).toContainLogEntry('UPDATE_ALIASES -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DONE'); + expect(records).toContainLogEntry('Migration completed'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts new file mode 100644 index 0000000000000..b321a400684b7 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/document_cleanup.test.ts @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay } from '../test_utils'; +import { + getBaseMigratorParams, + getDeletedType, + getExcludedType, + getFooType, + getBarType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'document_cleanup.test.log'); + +describe('ZDT upgrades - document cleanup', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getFooType(), getBarType(), getDeletedType(), getExcludedType()], + }); + await runMigrations(); + + const fooObjs = range(5).map((number) => ({ + id: `foo-${number}`, + type: 'foo', + attributes: { + someField: `foo_${number}`, + }, + })); + + const barObjs = range(5).map((number) => ({ + id: `bar-${number}`, + type: 'bar', + attributes: { + aKeyword: `bar_${number}`, + }, + })); + + const deletedObjs = range(5).map((number) => ({ + id: `server-${number}`, + type: 'server', + attributes: { + text: `some text`, + }, + })); + + const excludedObjs = range(5).map((number) => ({ + id: `excluded-${number}`, + type: 'excluded', + attributes: { + value: number, + }, + })); + + await savedObjectsRepository.bulkCreate([ + ...fooObjs, + ...barObjs, + ...deletedObjs, + ...excludedObjs, + ]); + }; + + it('deletes the documents', async () => { + await createBaseline(); + + const fooType = getFooType(); + const excludedType = getExcludedType(); + + fooType.modelVersions = { + ...fooType.modelVersions, + '3': dummyModelVersion, + }; + + const { runMigrations, client } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, excludedType], + }); + + await runMigrations(); + + const indexContent = await client.search<{ type: string }>({ index: '.kibana_1', size: 100 }); + + // normal type + expect(countResultsByType(indexContent, 'foo')).toEqual(5); + // unknown type + expect(countResultsByType(indexContent, 'bar')).toEqual(0); + // deleted type + expect(countResultsByType(indexContent, 'server')).toEqual(0); + // excludeOnUpgrade type + expect(countResultsByType(indexContent, 'excluded')).toEqual(3); + }); +}); + +const countResultsByType = ( + indexContents: SearchResponse<{ type: string }>, + type: string +): number => { + return indexContents.hits.hits.filter((result) => result._source?.type === type).length; +}; diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts index eadd24bf447ac..1674bf747f7b1 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/mapping_version_conflict.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'mapping_version_conflict.test.log'); @@ -35,15 +31,7 @@ describe('ZDT upgrades - mapping model version conflict', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; + const baseMigratorParams = getBaseMigratorParams(); beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); @@ -115,25 +103,16 @@ describe('ZDT upgrades - mapping model version conflict', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappingMeta.mappingVersions).toEqual({ foo: 2, bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; - - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; + const records = await parseLogFile(logFilePath); - // - expectLogsContains('Mappings model version check result: conflict'); - expectLogsContains('INIT -> FATAL'); + expect(records).toContainLogEntry('Mappings model version check result: conflict'); + expect(records).toContainLogEntry('INIT -> FATAL'); }); }); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts new file mode 100644 index 0000000000000..2be7c93965693 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/rerun_same_version.test.ts @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType } from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'rerun_same_version.test.log'); + +describe('ZDT upgrades - rerun migration on same version', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const fooType = getFooType(); + const barType = getBarType(); + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [fooType, barType], + }); + await runMigrations(); + }; + + it('should perform a no-op upgrade', async () => { + await createBaseline(); + + const fooType = getFooType(); + const barType = getBarType(); + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [fooType, barType], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('-> DONE'); + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts new file mode 100644 index 0000000000000..2fbbe7e090267 --- /dev/null +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/standard_workflow.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Path from 'path'; +import fs from 'fs/promises'; +import { range } from 'lodash'; +import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; +import { SavedObjectsBulkCreateObject } from '@kbn/core-saved-objects-api-server'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { + getBaseMigratorParams, + getSampleAType, + getSampleBType, + dummyModelVersion, +} from './base.fixtures'; + +export const logFilePath = Path.join(__dirname, 'standard_workflow.test.log'); + +describe('ZDT upgrades - basic document migration', () => { + let esServer: TestElasticsearchUtils['es']; + + const startElasticsearch = async () => { + const { startES } = createTestServers({ + adjustTimeout: (t: number) => jest.setTimeout(t), + settings: { + es: { + license: 'basic', + }, + }, + }); + return await startES(); + }; + + beforeAll(async () => { + await fs.unlink(logFilePath).catch(() => {}); + esServer = await startElasticsearch(); + }); + + afterAll(async () => { + await esServer?.stop(); + await delay(10); + }); + + const createBaseline = async () => { + const { runMigrations, savedObjectsRepository } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + types: [getSampleAType(), getSampleBType()], + }); + await runMigrations(); + + const sampleAObjs = range(5).map((number) => ({ + id: `a-${number}`, + type: 'sample_a', + attributes: { keyword: `a_${number}`, boolean: true }, + })); + + await savedObjectsRepository.bulkCreate(sampleAObjs); + + const sampleBObjs = range(5).map((number) => ({ + id: `b-${number}`, + type: 'sample_b', + attributes: { text: `i am number ${number}`, text2: `some static text` }, + })); + + await savedObjectsRepository.bulkCreate(sampleBObjs); + }; + + it('follows the expected stages and transitions', async () => { + await createBaseline(); + + const typeA = getSampleAType(); + const typeB = getSampleBType(); + + typeA.modelVersions = { + ...typeA.modelVersions, + '2': dummyModelVersion, + }; + + typeB.modelVersions = { + ...typeB.modelVersions, + '2': dummyModelVersion, + }; + + const { runMigrations } = await getKibanaMigratorTestKit({ + ...getBaseMigratorParams(), + logFilePath, + types: [typeA, typeB], + }); + + await runMigrations(); + + const records = await parseLogFile(logFilePath); + + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('INDEX_STATE_UPDATE_DONE -> DOCUMENTS_UPDATE_INIT'); + expect(records).toContainLogEntry('DOCUMENTS_UPDATE_INIT -> SET_DOC_MIGRATION_STARTED'); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED -> SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry( + 'SET_DOC_MIGRATION_STARTED_WAIT_FOR_INSTANCES -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS -> CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'CLEANUP_UNKNOWN_AND_EXCLUDED_DOCS_WAIT_FOR_TASK -> OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_TRANSFORM' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_BULK_INDEX -> OUTDATED_DOCUMENTS_SEARCH_READ' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT -> OUTDATED_DOCUMENTS_SEARCH_REFRESH' + ); + expect(records).toContainLogEntry( + 'OUTDATED_DOCUMENTS_SEARCH_REFRESH -> UPDATE_DOCUMENT_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry( + 'UPDATE_DOCUMENT_MODEL_VERSIONS -> UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES' + ); + expect(records).toContainLogEntry('UPDATE_DOCUMENT_MODEL_VERSIONS_WAIT_FOR_INSTANCES -> DONE'); + + expect(records).toContainLogEntry('Migration completed'); + }); +}); diff --git a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts index 5d27c32c1ee6a..9829da66a965e 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/zero_downtime/update_mappings.test.ts @@ -8,15 +8,11 @@ import Path from 'path'; import fs from 'fs/promises'; -import JSON5 from 'json5'; -import { LogRecord } from '@kbn/logging'; import { createTestServers, type TestElasticsearchUtils } from '@kbn/core-test-helpers-kbn-server'; -import { - getKibanaMigratorTestKit, - type KibanaMigratorTestKitParams, -} from '../kibana_migrator_test_kit'; -import { delay } from '../test_utils'; -import { getFooType, getBarType, dummyModelVersion } from './base_types.fixtures'; +import '../jest_matchers'; +import { getKibanaMigratorTestKit } from '../kibana_migrator_test_kit'; +import { delay, parseLogFile } from '../test_utils'; +import { getBaseMigratorParams, getFooType, getBarType, dummyModelVersion } from './base.fixtures'; export const logFilePath = Path.join(__dirname, 'update_mappings.test.log'); @@ -35,16 +31,6 @@ describe('ZDT upgrades - basic mapping update', () => { return await startES(); }; - const baseMigratorParams: KibanaMigratorTestKitParams = { - kibanaIndex: '.kibana', - kibanaVersion: '8.7.0', - settings: { - migrations: { - algorithm: 'zdt', - }, - }, - }; - beforeAll(async () => { await fs.unlink(logFilePath).catch(() => {}); esServer = await startElasticsearch(); @@ -59,7 +45,7 @@ describe('ZDT upgrades - basic mapping update', () => { const fooType = getFooType(); const barType = getBarType(); const { runMigrations } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), types: [fooType, barType], }); await runMigrations(); @@ -91,7 +77,7 @@ describe('ZDT upgrades - basic mapping update', () => { }; const { runMigrations, client } = await getKibanaMigratorTestKit({ - ...baseMigratorParams, + ...getBaseMigratorParams(), logFilePath, types: [fooType, barType], }); @@ -115,7 +101,7 @@ describe('ZDT upgrades - basic mapping update', () => { const mappings = index.mappings ?? {}; const mappingMeta = mappings._meta ?? {}; - expect(aliases).toEqual(['.kibana', '.kibana_8.7.0']); + expect(aliases).toEqual(['.kibana', '.kibana_8.8.0']); expect(mappings.properties).toEqual( expect.objectContaining({ @@ -124,32 +110,21 @@ describe('ZDT upgrades - basic mapping update', () => { }) ); - expect(mappingMeta).toEqual({ - // doc migration not implemented yet - docVersions are not bumped. - docVersions: { - foo: 2, - bar: 1, - }, - mappingVersions: { - foo: 3, - bar: 2, - }, + expect(mappingMeta.mappingVersions).toEqual({ + foo: 3, + bar: 2, }); - const logFileContent = await fs.readFile(logFilePath, 'utf-8'); - const records = logFileContent - .split('\n') - .filter(Boolean) - .map((str) => JSON5.parse(str)) as LogRecord[]; + const records = await parseLogFile(logFilePath); - const expectLogsContains = (messagePrefix: string) => { - expect(records.find((entry) => entry.message.includes(messagePrefix))).toBeDefined(); - }; - - expectLogsContains('INIT -> UPDATE_INDEX_MAPPINGS'); - expectLogsContains('UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK'); - expectLogsContains('UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS'); - expectLogsContains('UPDATE_MAPPING_MODEL_VERSIONS -> DONE'); - expectLogsContains('Migration completed'); + expect(records).toContainLogEntry('INIT -> UPDATE_INDEX_MAPPINGS'); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS -> UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK' + ); + expect(records).toContainLogEntry( + 'UPDATE_INDEX_MAPPINGS_WAIT_FOR_TASK -> UPDATE_MAPPING_MODEL_VERSIONS' + ); + expect(records).toContainLogEntry('UPDATE_MAPPING_MODEL_VERSIONS -> INDEX_STATE_UPDATE_DONE'); + expect(records).toContainLogEntry('Migration completed'); }); }); diff --git a/test/accessibility/apps/console.ts b/test/accessibility/apps/console.ts index e619b4bbf5530..5bfc40952ffcb 100644 --- a/test/accessibility/apps/console.ts +++ b/test/accessibility/apps/console.ts @@ -12,7 +12,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'console']); const a11y = getService('a11y'); - describe('Dev tools console', () => { + // https://github.com/elastic/kibana/issues/148538 + describe.skip('Dev tools console', () => { before(async () => { await PageObjects.common.navigateToApp('console'); }); diff --git a/test/accessibility/apps/dashboard.ts b/test/accessibility/apps/dashboard.ts index 3db8b99c577a4..b334563167fcd 100644 --- a/test/accessibility/apps/dashboard.ts +++ b/test/accessibility/apps/dashboard.ts @@ -109,12 +109,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await a11y.testAppSnapshot(); }); - it('Test full screen', async () => { + // https://github.com/elastic/kibana/issues/153597 + it.skip('Test full screen', async () => { await PageObjects.dashboard.clickFullScreenMode(); await a11y.testAppSnapshot(); }); - it('Exit out of full screen mode', async () => { + // https://github.com/elastic/kibana/issues/153597 + it.skip('Exit out of full screen mode', async () => { await PageObjects.dashboard.exitFullScreenMode(); await a11y.testAppSnapshot(); }); diff --git a/test/accessibility/apps/discover.ts b/test/accessibility/apps/discover.ts index b2fa7946817e8..df59456ee5485 100644 --- a/test/accessibility/apps/discover.ts +++ b/test/accessibility/apps/discover.ts @@ -172,13 +172,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await testSubjects.click('unifiedHistogramChartOptionsToggle'); }); - it('a11y test for data grid sort panel', async () => { + // https://github.com/elastic/kibana/issues/148567 + it.skip('a11y test for data grid sort panel', async () => { await testSubjects.click('dataGridColumnSortingButton'); await a11y.testAppSnapshot(); await browser.pressKeys(browser.keys.ESCAPE); }); - - it('a11y test for setting row height for display panel', async () => { + // https://github.com/elastic/kibana/issues/148567 + it.skip('a11y test for setting row height for display panel', async () => { await testSubjects.click('dataGridDisplaySelectorPopover'); await a11y.testAppSnapshot(); await browser.pressKeys(browser.keys.ESCAPE); diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 04e31f9e0b641..37453e1364f8d 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { PostureTypes } from './types'; + export const STATUS_ROUTE_PATH = '/internal/cloud_security_posture/status'; export const STATS_ROUTE_PATH = '/internal/cloud_security_posture/stats/{policy_template}'; export const BENCHMARKS_ROUTE_PATH = '/internal/cloud_security_posture/benchmarks'; @@ -85,3 +87,10 @@ export const SUPPORTED_CLOUDBEAT_INPUTS = [ CLOUDBEAT_VULN_MGMT_GCP, CLOUDBEAT_VULN_MGMT_AZURE, ] as const; + +export const POSTURE_TYPES: { [x: string]: PostureTypes } = { + [KSPM_POLICY_TEMPLATE]: KSPM_POLICY_TEMPLATE, + [CSPM_POLICY_TEMPLATE]: CSPM_POLICY_TEMPLATE, + [VULN_MGMT_POLICY_TEMPLATE]: VULN_MGMT_POLICY_TEMPLATE, + [POSTURE_TYPE_ALL]: POSTURE_TYPE_ALL, +} as const; diff --git a/x-pack/plugins/cloud_security_posture/common/types.ts b/x-pack/plugins/cloud_security_posture/common/types.ts index 83fd9a4b276c1..3552a663866e1 100644 --- a/x-pack/plugins/cloud_security_posture/common/types.ts +++ b/x-pack/plugins/cloud_security_posture/common/types.ts @@ -12,7 +12,7 @@ import { CspRuleTemplateMetadata } from './schemas/csp_rule_template_metadata'; export type Evaluation = 'passed' | 'failed' | 'NA'; -export type PostureTypes = 'cspm' | 'kspm' | 'all'; +export type PostureTypes = 'cspm' | 'kspm' | 'vuln_mgmt' | 'all'; /** number between 1-100 */ export type Score = number; @@ -85,6 +85,7 @@ export interface BaseCspSetupStatus { latestPackageVersion: string; cspm: BaseCspSetupBothPolicy; kspm: BaseCspSetupBothPolicy; + vuln_mgmt: BaseCspSetupBothPolicy; isPluginInitialized: boolean; } diff --git a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts index c8d876a7281da..47ab871dc9d21 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, type Logger } from '@kbn/core/server'; -import { IndexStatus } from '../../common/types'; +import { IndexStatus, PostureTypes } from '../../common/types'; export const checkIndexStatus = async ( esClient: ElasticsearchClient, index: string, logger: Logger, - postureType: 'cspm' | 'kspm' | 'all' = 'all' + postureType: PostureTypes = 'all' ): Promise => { const query = postureType === 'all' diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts index 7f1345ced245f..5cf11f8d1f980 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts @@ -6,10 +6,10 @@ */ import { calculateCspStatusCode } from './status'; -import { CSPM_POLICY_TEMPLATE } from '../../../common/constants'; +import { CSPM_POLICY_TEMPLATE, VULN_MGMT_POLICY_TEMPLATE } from '../../../common/constants'; -describe('calculateCspStatusCode test', () => { - it('Verify status when there are no permission', async () => { +describe('calculateCspStatusCode for cspm', () => { + it('Verify status when there are no permission for cspm', async () => { const statusCode = calculateCspStatusCode( CSPM_POLICY_TEMPLATE, { @@ -145,3 +145,141 @@ describe('calculateCspStatusCode test', () => { expect(statusCode).toMatch('indexing'); }); }); + +describe('calculateCspStatusCode for vul_mgmt', () => { + it('Verify status when there are no permission for vul_mgmt', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'unprivileged', + findings: 'unprivileged', + score: 'unprivileged', + }, + 1, + 1, + 1, + ['cspm'] + ); + + expect(statusCode).toMatch('unprivileged'); + }); + + it('Verify status when there are no vul_mgmt findings, no healthy agents and no installed policy templates', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 0, + 0, + 0, + [] + ); + + expect(statusCode).toMatch('not-installed'); + }); + + it('Verify status when there are vul_mgmt findings and installed policies but no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are vul_mgmt findings ,installed policies and healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'not-empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexed'); + }); + + it('Verify status when there are no vul_mgmt findings ,installed policies and no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 9, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('waiting_for_results'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings and been more than 10 minutes', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 11, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('index-timeout'); + }); + + it('Verify status when there are installed policies, healthy agents past vul_mgmt findings but no recent findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 0, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexing'); + }); +}); diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts index 23578194422ee..e643741bc4320 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts @@ -17,10 +17,14 @@ import { LATEST_FINDINGS_INDEX_DEFAULT_NS, FINDINGS_INDEX_PATTERN, BENCHMARK_SCORE_INDEX_DEFAULT_NS, + VULNERABILITIES_INDEX_PATTERN, KSPM_POLICY_TEMPLATE, CSPM_POLICY_TEMPLATE, + POSTURE_TYPES, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + VULN_MGMT_POLICY_TEMPLATE, } from '../../../common/constants'; -import type { CspApiRequestHandlerContext, CspRouter } from '../../types'; +import type { CspApiRequestHandlerContext, CspRouter, StatusResponseInfo } from '../../types'; import type { CspSetupStatus, CspStatusCode, @@ -72,7 +76,7 @@ export const calculateCspStatusCode = ( indicesStatus: { findingsLatest: IndexStatus; findings: IndexStatus; - score: IndexStatus; + score?: IndexStatus; }, installedCspPackagePolicies: number, healthyAgents: number, @@ -80,8 +84,7 @@ export const calculateCspStatusCode = ( installedPolicyTemplates: string[] ): CspStatusCode => { // We check privileges only for the relevant indices for our pages to appear - const postureTypeCheck = - postureType === CSPM_POLICY_TEMPLATE ? CSPM_POLICY_TEMPLATE : KSPM_POLICY_TEMPLATE; + const postureTypeCheck: PostureTypes = POSTURE_TYPES[postureType]; if (indicesStatus.findingsLatest === 'unprivileged' || indicesStatus.score === 'unprivileged') return 'unprivileged'; if (!installedPolicyTemplates.includes(postureTypeCheck)) return 'not-installed'; @@ -133,10 +136,13 @@ const getCspStatus = async ({ findingsLatestIndexStatusKspm, findingsIndexStatusKspm, scoreIndexStatusKspm, + vulnerabilitiesLatestIndexStatus, + vulnerabilitiesIndexStatus, installation, latestCspPackage, installedPackagePoliciesKspm, installedPackagePoliciesCspm, + installedPackagePoliciesVulnMgmt, installedPolicyTemplates, ] = await Promise.all([ checkIndexStatus(esClient.asCurrentUser, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger), @@ -151,8 +157,22 @@ const getCspStatus = async ({ checkIndexStatus(esClient.asCurrentUser, FINDINGS_INDEX_PATTERN, logger, 'kspm'), checkIndexStatus(esClient.asCurrentUser, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'kspm'), + checkIndexStatus( + esClient.asCurrentUser, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + checkIndexStatus( + esClient.asCurrentUser, + VULNERABILITIES_INDEX_PATTERN, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + packageService.asInternalUser.getInstallation(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), packageService.asInternalUser.fetchFindLatestPackage(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), + getCspPackagePolicies( soClient, packagePolicyService, @@ -171,9 +191,17 @@ const getCspStatus = async ({ }, CSPM_POLICY_TEMPLATE ), + getCspPackagePolicies( + soClient, + packagePolicyService, + CLOUD_SECURITY_POSTURE_PACKAGE_NAME, + { + per_page: 10000, + }, + VULN_MGMT_POLICY_TEMPLATE + ), getInstalledPolicyTemplates(packagePolicyService, soClient), ]); - const healthyAgentsKspm = await getHealthyAgents( soClient, installedPackagePoliciesKspm.items, @@ -189,8 +217,18 @@ const getCspStatus = async ({ agentService, logger ); + + const healthyAgentsVulMgmt = await getHealthyAgents( + soClient, + installedPackagePoliciesVulnMgmt.items, + agentPolicyService, + agentService, + logger + ); const installedPackagePoliciesTotalKspm = installedPackagePoliciesKspm.total; const installedPackagePoliciesTotalCspm = installedPackagePoliciesCspm.total; + const installedPackagePoliciesTotalVulnMgmt = installedPackagePoliciesVulnMgmt.total; + const latestCspPackageVersion = latestCspPackage.version; const MIN_DATE = 0; @@ -207,6 +245,10 @@ const getCspStatus = async ({ index: BENCHMARK_SCORE_INDEX_DEFAULT_NS, status: scoreIndexStatus, }, + { + index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + status: vulnerabilitiesLatestIndexStatus, + }, ]; const statusCspm = calculateCspStatusCode( @@ -235,38 +277,38 @@ const getCspStatus = async ({ installedPolicyTemplates ); - if ((statusCspm && statusKspm) === 'not-installed') - return { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, - }, - indicesDetails, - latestPackageVersion: latestCspPackageVersion, - isPluginInitialized: isPluginInitialized(), - }; - - const response = { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, + const statusVulnMgmt = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: vulnerabilitiesLatestIndexStatus, + findings: vulnerabilitiesIndexStatus, }, + installedPackagePoliciesTotalVulnMgmt, + healthyAgentsVulMgmt, + calculateDiffFromNowInMinutes(installation?.install_started_at || MIN_DATE), + installedPolicyTemplates + ); + + const statusResponseInfo = getStatusResponse({ + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, indicesDetails, - latestPackageVersion: latestCspPackageVersion, - installedPackageVersion: installation?.install_version, + latestCspPackageVersion, isPluginInitialized: isPluginInitialized(), + }); + + if ((statusCspm && statusKspm && statusVulnMgmt) === 'not-installed') return statusResponseInfo; + + const response = { + ...statusResponseInfo, + installedPackageVersion: installation?.install_version, }; assertResponse(response, logger); @@ -316,3 +358,40 @@ export const defineGetCspStatusRoute = (router: CspRouter): void => } } ); + +const getStatusResponse = (statusResponseInfo: StatusResponseInfo) => { + const { + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, + indicesDetails, + latestCspPackageVersion, + isPluginInitialized, + }: StatusResponseInfo = statusResponseInfo; + return { + [CSPM_POLICY_TEMPLATE]: { + status: statusCspm, + healthyAgents: healthyAgentsCspm, + installedPackagePolicies: installedPackagePoliciesTotalCspm, + }, + [KSPM_POLICY_TEMPLATE]: { + status: statusKspm, + healthyAgents: healthyAgentsKspm, + installedPackagePolicies: installedPackagePoliciesTotalKspm, + }, + [VULN_MGMT_POLICY_TEMPLATE]: { + status: statusVulnMgmt, + healthyAgents: healthyAgentsVulMgmt, + installedPackagePolicies: installedPackagePoliciesTotalVulnMgmt, + }, + indicesDetails, + isPluginInitialized, + latestPackageVersion: latestCspPackageVersion, + }; +}; diff --git a/x-pack/plugins/cloud_security_posture/server/types.ts b/x-pack/plugins/cloud_security_posture/server/types.ts index 503a529f4a681..1b3c4fd840347 100644 --- a/x-pack/plugins/cloud_security_posture/server/types.ts +++ b/x-pack/plugins/cloud_security_posture/server/types.ts @@ -34,6 +34,7 @@ import type { import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import type { FleetStartContract, FleetRequestHandlerContext } from '@kbn/fleet-plugin/server'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import { CspStatusCode, IndexDetails } from '../common/types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface CspServerPluginSetup {} @@ -97,3 +98,18 @@ export type CspRequestHandler< * @internal */ export type CspRouter = IRouter; + +export interface StatusResponseInfo { + statusCspm: CspStatusCode; + statusKspm: CspStatusCode; + statusVulnMgmt: CspStatusCode; + healthyAgentsCspm: number; + healthyAgentsKspm: number; + healthyAgentsVulMgmt: number; + installedPackagePoliciesTotalKspm: number; + installedPackagePoliciesTotalCspm: number; + installedPackagePoliciesTotalVulnMgmt: number; + indicesDetails: IndexDetails[]; + latestCspPackageVersion: string; + isPluginInitialized: boolean; +} diff --git a/x-pack/plugins/enterprise_search/server/integrations.ts b/x-pack/plugins/enterprise_search/server/integrations.ts index 94b12b35d4120..1ae3de167d3f4 100644 --- a/x-pack/plugins/enterprise_search/server/integrations.ts +++ b/x-pack/plugins/enterprise_search/server/integrations.ts @@ -10,6 +10,8 @@ import type { IntegrationCategory } from '@kbn/custom-integrations-plugin/common import type { CustomIntegrationsPluginSetup } from '@kbn/custom-integrations-plugin/server'; import { i18n } from '@kbn/i18n'; +import { ConfigType } from '.'; + interface WorkplaceSearchIntegration { id: string; title: string; @@ -297,65 +299,70 @@ const workplaceSearchIntegrations: WorkplaceSearchIntegration[] = [ ]; export const registerEnterpriseSearchIntegrations = ( + config: ConfigType, http: HttpServiceSetup, customIntegrations: CustomIntegrationsPluginSetup ) => { - workplaceSearchIntegrations.forEach((integration) => { + if (config.canDeployEntSearch) { + workplaceSearchIntegrations.forEach((integration) => { + customIntegrations.registerCustomIntegration({ + uiInternalPath: `/app/enterprise_search/workplace_search/sources/add/${integration.id}`, + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + `/plugins/enterpriseSearch/assets/source_icons/${integration.id}.svg` + ), + }, + ], + isBeta: false, + shipper: 'enterprise_search', + ...integration, + }); + }); + customIntegrations.registerCustomIntegration({ - uiInternalPath: `/app/enterprise_search/workplace_search/sources/add/${integration.id}`, + id: 'app_search_json', + title: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonName', { + defaultMessage: 'JSON', + }), + description: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonDescription', { + defaultMessage: 'Search over your JSON data with App Search.', + }), + categories: ['upload_file'], + uiInternalPath: '/app/enterprise_search/app_search/engines/new?method=json', icons: [ { - type: 'svg', - src: http.basePath.prepend( - `/plugins/enterpriseSearch/assets/source_icons/${integration.id}.svg` - ), + type: 'eui', + src: 'logoAppSearch', }, ], - isBeta: false, shipper: 'enterprise_search', - ...integration, + isBeta: false, }); - }); + } - customIntegrations.registerCustomIntegration({ - id: 'app_search_json', - title: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonName', { - defaultMessage: 'JSON', - }), - description: i18n.translate('xpack.enterpriseSearch.appSearch.integrations.jsonDescription', { - defaultMessage: 'Search over your JSON data with App Search.', - }), - categories: ['upload_file'], - uiInternalPath: '/app/enterprise_search/app_search/engines/new?method=json', - icons: [ - { - type: 'eui', - src: 'logoAppSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); - - customIntegrations.registerCustomIntegration({ - id: 'web_crawler', - title: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerName', { - defaultMessage: 'Web crawler', - }), - description: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerDescription', { - defaultMessage: 'Add search to your website with the Enterprise Search web crawler.', - }), - categories: ['enterprise_search', 'website_search', 'web', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=crawler', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasWebCrawler) { + customIntegrations.registerCustomIntegration({ + id: 'web_crawler', + title: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerName', { + defaultMessage: 'Web crawler', + }), + description: i18n.translate('xpack.enterpriseSearch.integrations.webCrawlerDescription', { + defaultMessage: 'Add search to your website with the Enterprise Search web crawler.', + }), + categories: ['enterprise_search', 'website_search', 'web', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=crawler', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } customIntegrations.registerCustomIntegration({ id: 'api', @@ -377,265 +384,282 @@ export const registerEnterpriseSearchIntegrations = ( isBeta: false, }); - customIntegrations.registerCustomIntegration({ - id: 'build_a_connector', - title: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorName', { - defaultMessage: 'Build a connector', - }), - description: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorDescription', { - defaultMessage: 'Search over data stored on custom data sources with Enterprise Search.', - }), - categories: ['enterprise_search', 'custom', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasNativeConnectors) { + customIntegrations.registerCustomIntegration({ + id: 'native_connector', + title: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorName', + { + defaultMessage: 'Use a connector', + } + ), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorDescription', + { + defaultMessage: + 'Search over your data sources with a native Enterprise Search connector.', + } + ), + categories: ['elastic_stack', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'native_connector', - title: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorName', - { - defaultMessage: 'Use a connector', - } - ), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.nativeConnectorDescription', - { - defaultMessage: 'Search over your data sources with a native Enterprise Search connector.', - } - ), - categories: ['elastic_stack', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'eui', - src: 'logoEnterpriseSearch', - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'mongodb', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBName', { + defaultMessage: 'MongoDB', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBDescription', + { + defaultMessage: 'Search over your MongoDB content with Enterprise Search.', + } + ), + categories: ['datastore', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mongodb.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'mongodb', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBName', { - defaultMessage: 'MongoDB', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.mongoDBDescription', - { - defaultMessage: 'Search over your MongoDB content with Enterprise Search.', - } - ), - categories: ['datastore', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mongodb.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'mysql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mysqlName', { + defaultMessage: 'MySQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.mysqlDescription', + { + defaultMessage: 'Search over your MySQL content with Enterprise Search.', + } + ), + categories: ['datastore', 'enterprise_search'], + uiInternalPath: + '/app/enterprise_search/content/search_indices/new_index?method=native_connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mysql.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } - customIntegrations.registerCustomIntegration({ - id: 'mysql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.mysqlName', { - defaultMessage: 'MySQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.mysqlDescription', - { - defaultMessage: 'Search over your MySQL content with Enterprise Search.', - } - ), - categories: ['datastore', 'enterprise_search'], - uiInternalPath: - '/app/enterprise_search/content/search_indices/new_index?method=native_connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/mysql.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + if (config.hasConnectors) { + customIntegrations.registerCustomIntegration({ + id: 'build_a_connector', + title: i18n.translate('xpack.enterpriseSearch.integrations.buildAConnectorName', { + defaultMessage: 'Build a connector', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.integrations.buildAConnectorDescription', + { + defaultMessage: 'Search over data stored on custom data sources with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'eui', + src: 'logoEnterpriseSearch', + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'postgresql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.postgresqlName', { - defaultMessage: 'PostgreSQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.postgreSQLDescription', - { - defaultMessage: 'Search over your content on PostgreSQL with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/postgresql.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'postgresql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.postgresqlName', { + defaultMessage: 'PostgreSQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.postgreSQLDescription', + { + defaultMessage: 'Search over your content on PostgreSQL with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/postgresql.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'oracle', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.oracleName', { - defaultMessage: 'Oracle', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.oracleDescription', - { - defaultMessage: 'Search over your content on Oracle with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/oracle.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'oracle', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.oracleName', { + defaultMessage: 'Oracle', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.oracleDescription', + { + defaultMessage: 'Search over your content on Oracle with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/oracle.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'ms_sql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.msSqlName', { - defaultMessage: 'Microsoft SQL', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.msSqlDescription', - { - defaultMessage: 'Search over your content on Microsoft SQL Server with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'custom', 'elastic_stack', 'datastore'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/microsoft_sql.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'ms_sql', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.msSqlName', { + defaultMessage: 'Microsoft SQL', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.msSqlDescription', + { + defaultMessage: + 'Search over your content on Microsoft SQL Server with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack', 'datastore'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/microsoft_sql.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'ms_sql', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.networkDriveName', { - defaultMessage: 'Network Drive', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.netowkrDriveDescription', - { - defaultMessage: 'Search over your Network Drive content with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'custom', 'elastic_stack', 'file_storage'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/network_drive.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'ms_sql', + title: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.networkDriveName', + { + defaultMessage: 'Network Drive', + } + ), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.netowkrDriveDescription', + { + defaultMessage: 'Search over your Network Drive content with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'custom', 'elastic_stack', 'file_storage'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/network_drive.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'amazon_s3', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.s3', { - defaultMessage: 'Amazon S3', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.s3Description', - { - defaultMessage: 'Search over your content on Amazon S3 with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'datastore', 'elastic_stack'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/amazon_s3.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'amazon_s3', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.s3', { + defaultMessage: 'Amazon S3', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.s3Description', + { + defaultMessage: 'Search over your content on Amazon S3 with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'datastore', 'elastic_stack'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/amazon_s3.svg'), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'google_cloud_storage', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.googleCloud', { - defaultMessage: 'Google Cloud Storage', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.googleCloudDescription', - { - defaultMessage: 'Search over your content on Google Cloud Storage with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend( - '/plugins/enterpriseSearch/assets/source_icons/google_cloud.svg' - ), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'google_cloud_storage', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.googleCloud', { + defaultMessage: 'Google Cloud Storage', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.googleCloudDescription', + { + defaultMessage: + 'Search over your content on Google Cloud Storage with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/google_cloud.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); - customIntegrations.registerCustomIntegration({ - id: 'azure_blob_storage', - title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.azureBlob', { - defaultMessage: 'Azure Blob Storage', - }), - description: i18n.translate( - 'xpack.enterpriseSearch.workplaceSearch.integrations.azureBlobDescription', - { - defaultMessage: 'Search over your content on Azure Blob Storage with Enterprise Search.', - } - ), - categories: ['enterprise_search', 'elastic_stack', 'custom'], - uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', - icons: [ - { - type: 'svg', - src: http.basePath.prepend('/plugins/enterpriseSearch/assets/source_icons/azure_blob.svg'), - }, - ], - shipper: 'enterprise_search', - isBeta: false, - }); + customIntegrations.registerCustomIntegration({ + id: 'azure_blob_storage', + title: i18n.translate('xpack.enterpriseSearch.workplaceSearch.integrations.azureBlob', { + defaultMessage: 'Azure Blob Storage', + }), + description: i18n.translate( + 'xpack.enterpriseSearch.workplaceSearch.integrations.azureBlobDescription', + { + defaultMessage: 'Search over your content on Azure Blob Storage with Enterprise Search.', + } + ), + categories: ['enterprise_search', 'elastic_stack', 'custom'], + uiInternalPath: '/app/enterprise_search/content/search_indices/new_index?method=connector', + icons: [ + { + type: 'svg', + src: http.basePath.prepend( + '/plugins/enterpriseSearch/assets/source_icons/azure_blob.svg' + ), + }, + ], + shipper: 'enterprise_search', + isBeta: false, + }); + } }; diff --git a/x-pack/plugins/enterprise_search/server/plugin.ts b/x-pack/plugins/enterprise_search/server/plugin.ts index 14a46bdcb6595..729258fe4e97c 100644 --- a/x-pack/plugins/enterprise_search/server/plugin.ts +++ b/x-pack/plugins/enterprise_search/server/plugin.ts @@ -136,7 +136,7 @@ export class EnterpriseSearchPlugin implements Plugin { ]; if (customIntegrations) { - registerEnterpriseSearchIntegrations(http, customIntegrations); + registerEnterpriseSearchIntegrations(config, http, customIntegrations); } /* diff --git a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts index 996e556ca1704..dd809258ef0c0 100644 --- a/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts +++ b/x-pack/plugins/maps/common/descriptor_types/data_request_descriptor_types.ts @@ -7,6 +7,7 @@ /* eslint-disable @typescript-eslint/consistent-type-definitions */ +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import type { Filter } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; @@ -33,6 +34,7 @@ export type DataFilters = { zoom: number; isReadOnly: boolean; joinKeyFilter?: Filter; + executionContext: KibanaExecutionContext; }; export type VectorSourceRequestMeta = DataFilters & { diff --git a/x-pack/plugins/maps/common/execution_context.test.ts b/x-pack/plugins/maps/common/execution_context.test.ts deleted file mode 100644 index 25e3813a7e8f0..0000000000000 --- a/x-pack/plugins/maps/common/execution_context.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { makeExecutionContext } from './execution_context'; - -describe('makeExecutionContext', () => { - test('returns basic fields if nothing is provided', () => { - const context = makeExecutionContext({}); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - }); - }); - - test('merges in context', () => { - const context = makeExecutionContext({ id: '123' }); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - id: '123', - }); - }); - - test('omits undefined values', () => { - const context = makeExecutionContext({ id: '123', description: undefined }); - expect(context).toStrictEqual({ - name: 'maps', - type: 'application', - id: '123', - }); - }); -}); diff --git a/x-pack/plugins/maps/common/execution_context.ts b/x-pack/plugins/maps/common/execution_context.ts deleted file mode 100644 index f62f1da85f99d..0000000000000 --- a/x-pack/plugins/maps/common/execution_context.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { isUndefined, omitBy } from 'lodash'; -import type { KibanaExecutionContext } from '@kbn/core/public'; -import { APP_ID } from './constants'; - -export function makeExecutionContext(context: { - id?: string; - url?: string; - description?: string; -}): KibanaExecutionContext { - return omitBy( - { - name: APP_ID, - type: 'application', - ...context, - }, - isUndefined - ); -} diff --git a/x-pack/plugins/maps/public/actions/map_action_constants.ts b/x-pack/plugins/maps/public/actions/map_action_constants.ts index aabca948ec80a..483615e44c1d0 100644 --- a/x-pack/plugins/maps/public/actions/map_action_constants.ts +++ b/x-pack/plugins/maps/public/actions/map_action_constants.ts @@ -39,6 +39,7 @@ export const REMOVE_TRACKED_LAYER_STATE = 'REMOVE_TRACKED_LAYER_STATE'; export const SET_OPEN_TOOLTIPS = 'SET_OPEN_TOOLTIPS'; export const UPDATE_DRAW_STATE = 'UPDATE_DRAW_STATE'; export const UPDATE_EDIT_STATE = 'UPDATE_EDIT_STATE'; +export const SET_EXECUTION_CONTEXT = 'SET_EXECUTION_CONTEXT'; export const SET_MAP_INIT_ERROR = 'SET_MAP_INIT_ERROR'; export const SET_WAITING_FOR_READY_HIDDEN_LAYERS = 'SET_WAITING_FOR_READY_HIDDEN_LAYERS'; export const SET_MAP_SETTINGS = 'SET_MAP_SETTINGS'; diff --git a/x-pack/plugins/maps/public/actions/map_actions.ts b/x-pack/plugins/maps/public/actions/map_actions.ts index e51a652359168..7ae6a56900400 100644 --- a/x-pack/plugins/maps/public/actions/map_actions.ts +++ b/x-pack/plugins/maps/public/actions/map_actions.ts @@ -11,6 +11,7 @@ import { AnyAction, Dispatch } from 'redux'; import { ThunkDispatch } from 'redux-thunk'; import turfBboxPolygon from '@turf/bbox-polygon'; import turfBooleanContains from '@turf/boolean-contains'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import type { Query, TimeRange } from '@kbn/es-query'; import { Geometry, Position } from 'geojson'; @@ -44,6 +45,7 @@ import { MAP_READY, ROLLBACK_MAP_SETTINGS, SET_EMBEDDABLE_SEARCH_CONTEXT, + SET_EXECUTION_CONTEXT, SET_GOTO, SET_MAP_INIT_ERROR, SET_MAP_SETTINGS, @@ -357,6 +359,13 @@ export function setEmbeddableSearchContext({ }; } +export function setExecutionContext(executionContext: KibanaExecutionContext) { + return { + type: SET_EXECUTION_CONTEXT, + executionContext, + }; +} + export function updateDrawState(drawState: DrawState | null) { return (dispatch: Dispatch) => { if (drawState !== null) { diff --git a/x-pack/plugins/maps/public/classes/joins/inner_join.ts b/x-pack/plugins/maps/public/classes/joins/inner_join.ts index b3a55ea55decd..a754650cdef80 100644 --- a/x-pack/plugins/maps/public/classes/joins/inner_join.ts +++ b/x-pack/plugins/maps/public/classes/joins/inner_join.ts @@ -5,6 +5,7 @@ * 2.0. */ +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/es-query'; import { Feature, GeoJsonProperties } from 'geojson'; import { ESTermSource } from '../sources/es_term_source'; @@ -137,8 +138,11 @@ export class InnerJoin { return this._descriptor; } - async getTooltipProperties(properties: GeoJsonProperties) { - return await this.getRightJoinSource().getTooltipProperties(properties); + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ) { + return await this.getRightJoinSource().getTooltipProperties(properties, executionContext); } getIndexPatternIds() { diff --git a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts index ef47418736792..1160a8a1aa764 100644 --- a/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts +++ b/x-pack/plugins/maps/public/classes/layers/__fixtures__/mock_sync_context.ts @@ -34,6 +34,7 @@ export class MockSyncContext implements DataRequestContext { }, zoom: 0, isReadOnly: false, + executionContext: {}, ...dataFilters, }; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts b/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts index d7382782d1af7..b46353fcef933 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/bounds_data.ts @@ -45,6 +45,7 @@ export async function syncBoundsData({ applyGlobalQuery: source.getApplyGlobalQuery(), applyGlobalTime: source.getApplyGlobalTime(), isFeatureEditorOpenForLayer, + executionContext: dataFilters.executionContext, }; let bounds = null; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx index 13a093ad59713..bff6a297fcd74 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx @@ -8,6 +8,7 @@ import React from 'react'; import { v4 as uuidv4 } from 'uuid'; import type { FilterSpecification, Map as MbMap, LayerSpecification } from '@kbn/mapbox-gl'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import { Feature, GeoJsonProperties, Geometry, Position } from 'geojson'; import _ from 'lodash'; @@ -94,7 +95,10 @@ export interface IVectorLayer extends ILayer { getSource(): IVectorSource; getFeatureId(feature: Feature): string | number | undefined; getFeatureById(id: string | number): Feature | null; - getPropertiesForTooltip(properties: GeoJsonProperties): Promise; + getPropertiesForTooltip( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; hasJoins(): boolean; showJoinEditor(): boolean; canShowTooltip(): boolean; @@ -466,6 +470,7 @@ export class AbstractVectorLayer extends AbstractLayer implements IVectorLayer { timeFilters: nextMeta.timeFilters, searchSessionId: dataFilters.searchSessionId, inspectorAdapters, + executionContext: dataFilters.executionContext, }); stopLoading(dataRequestId, requestToken, styleMeta, nextMeta); @@ -931,13 +936,19 @@ export class AbstractVectorLayer extends AbstractLayer implements IVectorLayer { } } - async getPropertiesForTooltip(properties: GeoJsonProperties) { + async getPropertiesForTooltip( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ) { const vectorSource = this.getSource(); - let allProperties = await vectorSource.getTooltipProperties(properties); + let allProperties = await vectorSource.getTooltipProperties(properties, executionContext); this._addJoinsToSourceTooltips(allProperties); for (let i = 0; i < this.getJoins().length; i++) { - const propsFromJoin = await this.getJoins()[i].getTooltipProperties(properties); + const propsFromJoin = await this.getJoins()[i].getTooltipProperties( + properties, + executionContext + ); allProperties = [...allProperties, ...propsFromJoin]; } return allProperties; diff --git a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx index 70d5a9c54cc92..bc1289e10404f 100644 --- a/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/ems_file_source/ems_file_source.tsx @@ -7,7 +7,7 @@ import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; -import { Feature } from 'geojson'; +import { Feature, GeoJsonProperties } from 'geojson'; import { FileLayer } from '@elastic/ems-client'; import { ImmutableSourceProperty, SourceEditorArgs } from '../source'; import { AbstractVectorSource, GeoJsonWithMeta, IVectorSource } from '../vector_source'; @@ -199,7 +199,7 @@ export class EMSFileSource extends AbstractVectorSource implements IEmsFileSourc return this._tooltipFields.length > 0; } - async getTooltipProperties(properties: unknown): Promise { + async getTooltipProperties(properties: GeoJsonProperties): Promise { const promises = this._tooltipFields.map((field) => { // @ts-ignore const value = properties[field.getName()]; diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts index a7d0b4c9beb8f..65325fed5d957 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.test.ts @@ -8,13 +8,14 @@ import { coreMock } from '@kbn/core/public/mocks'; import { MapExtent, VectorSourceRequestMeta } from '../../../../common/descriptor_types'; import { - getExecutionContext, + getExecutionContextService, getHttp, getIndexPatternService, getSearchService, } from '../../../kibana_services'; import { ESGeoGridSource } from './es_geo_grid_source'; import { + APP_ID, ES_GEO_FIELD_TYPE, GRID_RESOLUTION, RENDER_AS, @@ -139,7 +140,7 @@ describe('ESGeoGridSource', () => { name: 'some-app', }); // @ts-expect-error - getExecutionContext.mockReturnValue(coreStartMock.executionContext); + getExecutionContextService.mockReturnValue(coreStartMock.executionContext); }); afterEach(() => { @@ -177,6 +178,7 @@ describe('ESGeoGridSource', () => { zoom: 0, isForceRefresh: false, isFeatureEditorOpenForLayer: false, + executionContext: { name: APP_ID }, }; describe('getGeoJsonWithMeta', () => { @@ -310,9 +312,36 @@ describe('ESGeoGridSource', () => { it('getTileUrl', async () => { const tileUrl = await mvtGeogridSource.getTileUrl(vectorSourceRequestMeta, '1234', false, 5); - expect(tileUrl).toEqual( - "rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=foo-*&gridPrecision=8&hasLabels=false&buffer=5&requestBody=(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'1'%3A('0'%3Asize%2C'1'%3A0)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A''))%2C'6'%3A('0'%3Aaggs%2C'1'%3A())))&renderAs=heatmap&token=1234" + const urlParts = tileUrl.split('?'); + expect(urlParts[0]).toEqual('rootdir/api/maps/mvt/getGridTile/{z}/{x}/{y}.pbf'); + + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params)).toEqual({ + buffer: '5', + geometryFieldName: 'bar', + gridPrecision: '8', + hasLabels: 'false', + index: 'foo-*', + renderAs: 'heatmap', + requestBody: + "(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'1'%3A('0'%3Asize%2C'1'%3A0)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A''))%2C'6'%3A('0'%3Aaggs%2C'1'%3A())))", + token: '1234', + }); + }); + + it('getTileUrl should include executionContextId when provided', async () => { + const tileUrl = await mvtGeogridSource.getTileUrl( + { + ...vectorSourceRequestMeta, + executionContext: { name: APP_ID, id: 'map1234' }, + }, + '1234', + false, + 5 ); + const urlParts = tileUrl.split('?'); + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params).executionContextId).toEqual('map1234'); }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx index dea665f904b01..c19e715326bc3 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_grid_source/es_geo_grid_source.tsx @@ -10,6 +10,7 @@ import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; import { Feature } from 'geojson'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { ISearchSource } from '@kbn/data-plugin/common/search/search_source'; import { DataView } from '@kbn/data-plugin/common'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; @@ -50,7 +51,7 @@ import { } from '../../../../common/descriptor_types'; import { ImmutableSourceProperty, OnSourceChangeArgs, SourceEditorArgs } from '../source'; import { isValidStringConfig } from '../../util/valid_string_config'; -import { makePublicExecutionContext } from '../../../util'; +import { getExecutionContextId, mergeExecutionContext } from '../execution_context_utils'; import { isMvt } from './is_mvt'; import { VectorStyle } from '../../styles/vector/vector_style'; import { getIconSize } from './get_icon_size'; @@ -256,6 +257,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive, bufferedExtent, inspectorAdapters, + executionContext, }: { searchSource: ISearchSource; searchSessionId?: string; @@ -267,6 +269,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive: () => boolean; bufferedExtent: MapExtent; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }) { const gridsPerRequest: number = Math.floor(DEFAULT_MAX_BUCKETS_LIMIT / bucketsPerGrid); const aggs: any = { @@ -337,7 +340,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo } ), searchSessionId, - executionContext: makePublicExecutionContext('es_geo_grid_source:cluster_composite'), + executionContext: mergeExecutionContext( + { description: 'es_geo_grid_source:cluster_composite' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -365,6 +371,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent, tooManyBuckets, inspectorAdapters, + executionContext, }: { searchSource: ISearchSource; searchSessionId?: string; @@ -375,6 +382,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent: MapExtent; tooManyBuckets: boolean; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise { const valueAggsDsl = tooManyBuckets ? this.getValueAggsDsl(indexPattern, (metric) => { @@ -411,7 +419,10 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo defaultMessage: 'Elasticsearch geo grid aggregation request', }), searchSessionId, - executionContext: makePublicExecutionContext('es_geo_grid_source:cluster'), + executionContext: mergeExecutionContext( + { description: 'es_geo_grid_source:cluster' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -471,6 +482,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo isRequestStillActive, bufferedExtent: searchFilters.buffer, inspectorAdapters, + executionContext: searchFilters.executionContext, }) : await this._nonCompositeAggRequest({ searchSource, @@ -482,6 +494,7 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo bufferedExtent: searchFilters.buffer, tooManyBuckets, inspectorAdapters, + executionContext: searchFilters.executionContext, }); return { @@ -513,15 +526,21 @@ export class ESGeoGridSource extends AbstractESAggSource implements IMvtVectorSo `/${GIS_API_PATH}/${MVT_GETGRIDTILE_API_PATH}/{z}/{x}/{y}.pbf` ); - return `${mvtUrlServicePath}\ -?geometryFieldName=${this._descriptor.geoField}\ -&index=${dataView.getIndexPattern()}\ -&gridPrecision=${this._getGeoGridPrecisionResolutionDelta()}\ -&hasLabels=${hasLabels}\ -&buffer=${buffer}\ -&requestBody=${encodeMvtResponseBody(searchSource.getSearchRequestBody())}\ -&renderAs=${this._descriptor.requestType}\ -&token=${refreshToken}`; + const params = new URLSearchParams(); + params.set('geometryFieldName', this._descriptor.geoField); + params.set('index', dataView.getIndexPattern()); + params.set('gridPrecision', this._getGeoGridPrecisionResolutionDelta().toString()); + params.set('hasLabels', hasLabels.toString()); + params.set('buffer', buffer.toString()); + params.set('requestBody', encodeMvtResponseBody(searchSource.getSearchRequestBody())); + params.set('renderAs', this._descriptor.requestType); + params.set('token', refreshToken); + const executionContextId = getExecutionContextId(searchFilters.executionContext); + if (executionContextId) { + params.set('executionContextId', executionContextId); + } + + return `${mvtUrlServicePath}?${params.toString()}`; } isFilterByMapBounds(): boolean { diff --git a/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx index 78aae064a6555..33b2b2e38317b 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_geo_line_source/es_geo_line_source.tsx @@ -38,7 +38,7 @@ import { IField } from '../../fields/field'; import { ITooltipProperty, TooltipProperty } from '../../tooltips/tooltip_property'; import { getIsGoldPlus } from '../../../licensed_features'; import { LICENSED_FEATURES } from '../../../licensed_features'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; type ESGeoLineSourceSyncMeta = Pick; @@ -218,7 +218,10 @@ export class ESGeoLineSource extends AbstractESAggSource { defaultMessage: 'Elasticsearch terms request to fetch entities within map buffer.', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_geo_line:entities'), + executionContext: mergeExecutionContext( + { description: 'es_geo_line:entities' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); const entityBuckets: Array<{ key: string; doc_count: number }> = _.get( @@ -291,7 +294,10 @@ export class ESGeoLineSource extends AbstractESAggSource { 'Elasticsearch geo_line request to fetch tracks for entities. Tracks are not filtered by map buffer.', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_geo_line:tracks'), + executionContext: mergeExecutionContext( + { description: 'es_geo_line:tracks' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); const { featureCollection, numTrimmedTracks } = convertToGeoJson( diff --git a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx index dcec30b1f71ac..ae00e00e58ae6 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_pew_pew_source/es_pew_pew_source.tsx @@ -26,7 +26,7 @@ import { AbstractESAggSource } from '../es_agg_source'; import { registerSource } from '../source_registry'; import { turfBboxToBounds } from '../../../../common/elasticsearch_util'; import { DataRequestAbortError } from '../../util/data_request'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; import { SourceEditorArgs } from '../source'; import { ESPewPewSourceDescriptor, @@ -184,7 +184,10 @@ export class ESPewPewSource extends AbstractESAggSource { defaultMessage: 'Source-destination connections request', }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_pew_pew_source:connections'), + executionContext: mergeExecutionContext( + { description: 'es_pew_pew_source:connections' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -229,7 +232,10 @@ export class ESPewPewSource extends AbstractESAggSource { searchSource.fetch$({ abortSignal: abortController.signal, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_pew_pew_source:bounds'), + executionContext: mergeExecutionContext( + { description: 'es_pew_pew_source:bounds' }, + boundsFilters.executionContext + ), }) ); const destBounds = (esResp.aggregations?.destFitToBounds as AggregationsGeoBoundsAggregate) diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts index 2e9fa9e027f5a..5157848cc5765 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { ES_GEO_FIELD_TYPE, SCALING_TYPES } from '../../../../common/constants'; +import { APP_ID, ES_GEO_FIELD_TYPE, SCALING_TYPES } from '../../../../common/constants'; jest.mock('../../../kibana_services'); jest.mock('./util/load_index_settings'); @@ -108,17 +108,48 @@ describe('ESSearchSource', () => { applyForceRefresh: true, isForceRefresh: false, isFeatureEditorOpenForLayer: false, + executionContext: { name: APP_ID }, }; - it('Should only include required props', async () => { + it('should include required props', async () => { const esSearchSource = new ESSearchSource({ geoField: geoFieldName, indexPatternId: 'ipId', }); const tileUrl = await esSearchSource.getTileUrl(searchFilters, '1234', false, 5); - expect(tileUrl).toBe( - `rootdir/api/maps/mvt/getTile/{z}/{x}/{y}.pbf?geometryFieldName=bar&index=foobar-title-*&hasLabels=false&buffer=5&requestBody=(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'1'%3A('0'%3Asize%2C'1'%3A1000)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A'tooltipField%3A%20foobar'))%2C'6'%3A('0'%3AfieldsFromSource%2C'1'%3A!(_id))%2C'7'%3A('0'%3Asource%2C'1'%3A!f)%2C'8'%3A('0'%3Afields%2C'1'%3A!(tooltipField%2CstyleField))))&token=1234` + + const urlParts = tileUrl.split('?'); + expect(urlParts[0]).toEqual('rootdir/api/maps/mvt/getTile/{z}/{x}/{y}.pbf'); + + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params)).toEqual({ + buffer: '5', + geometryFieldName: 'bar', + hasLabels: 'false', + index: 'foobar-title-*', + requestBody: + "(foobar%3AES_DSL_PLACEHOLDER%2Cparams%3A('0'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'1'%3A('0'%3Asize%2C'1'%3A1000)%2C'2'%3A('0'%3Afilter%2C'1'%3A!())%2C'3'%3A('0'%3Aquery)%2C'4'%3A('0'%3Aindex%2C'1'%3A(fields%3A()%2Ctitle%3A'foobar-title-*'))%2C'5'%3A('0'%3Aquery%2C'1'%3A(language%3AKQL%2Cquery%3A'tooltipField%3A%20foobar'))%2C'6'%3A('0'%3AfieldsFromSource%2C'1'%3A!(_id))%2C'7'%3A('0'%3Asource%2C'1'%3A!f)%2C'8'%3A('0'%3Afields%2C'1'%3A!(tooltipField%2CstyleField))))", + token: '1234', + }); + }); + + it('should include executionContextId when provided', async () => { + const esSearchSource = new ESSearchSource({ + geoField: geoFieldName, + indexPatternId: 'ipId', + }); + const tileUrl = await esSearchSource.getTileUrl( + { + ...searchFilters, + executionContext: { name: APP_ID, id: 'map1234' }, + }, + '1234', + false, + 5 ); + const urlParts = tileUrl.split('?'); + const params = new URLSearchParams(urlParts[1]); + expect(Object.fromEntries(params).executionContextId).toEqual('map1234'); }); }); }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx index c5ed07a4c0e18..3b8ab273fbbc6 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/es_search_source/es_search_source.tsx @@ -9,6 +9,7 @@ import _ from 'lodash'; import React, { ReactElement } from 'react'; import { i18n } from '@kbn/i18n'; import { GeoJsonProperties, Geometry, Position } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { type Filter, buildPhraseFilter, type TimeRange } from '@kbn/es-query'; import type { DataViewField, DataView } from '@kbn/data-plugin/common'; import { lastValueFrom } from 'rxjs'; @@ -74,7 +75,7 @@ import { getIsDrawLayer, getMatchingIndexes, } from './util/feature_edit'; -import { makePublicExecutionContext } from '../../../util'; +import { getExecutionContextId, mergeExecutionContext } from '../execution_context_utils'; import { FeatureGeometryFilterForm } from '../../../connected_components/mb_map/tooltip_control/features_tooltip'; type ESSearchSourceSyncMeta = Pick< @@ -354,7 +355,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource registerCancelCallback, requestDescription: 'Elasticsearch document top hits request', searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_search_source:top_hits'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:top_hits' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -438,7 +442,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource registerCancelCallback, requestDescription: 'Elasticsearch document request', searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_search_source:doc_search'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:doc_search' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); @@ -574,7 +581,12 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource return this._tooltipFields.length > 0; } - async _loadTooltipProperties(docId: string | number, index: string, indexPattern: DataView) { + async _loadTooltipProperties( + docId: string | number, + index: string, + indexPattern: DataView, + executionContext: KibanaExecutionContext + ) { if (this._tooltipFields.length === 0) { return {}; } @@ -616,7 +628,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource const { rawResponse: resp } = await lastValueFrom( searchSource.fetch$({ legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_search_source:load_tooltip_properties'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:load_tooltip_properties' }, + executionContext + ), }) ); @@ -643,7 +658,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource return this._tooltipFields.map((field: IField) => field.getName()); } - async getTooltipProperties(properties: GeoJsonProperties): Promise { + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise { if (properties === null) { throw new Error('properties cannot be null'); } @@ -651,7 +669,8 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource const propertyValues = await this._loadTooltipProperties( properties._id, properties._index, - indexPattern + indexPattern, + executionContext ); const tooltipProperties = this._tooltipFields.map((field) => { const value = propertyValues[field.getName()]; @@ -869,13 +888,19 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource delete requestBody.script_fields; delete requestBody.stored_fields; - return `${mvtUrlServicePath}\ -?geometryFieldName=${this._descriptor.geoField}\ -&index=${dataView.getIndexPattern()}\ -&hasLabels=${hasLabels}\ -&buffer=${buffer}\ -&requestBody=${encodeMvtResponseBody(requestBody)}\ -&token=${refreshToken}`; + const params = new URLSearchParams(); + params.set('geometryFieldName', this._descriptor.geoField); + params.set('index', dataView.getIndexPattern()); + params.set('hasLabels', hasLabels.toString()); + params.set('buffer', buffer.toString()); + params.set('requestBody', encodeMvtResponseBody(requestBody)); + params.set('token', refreshToken); + const executionContextId = getExecutionContextId(searchFilters.executionContext); + if (executionContextId) { + params.set('executionContextId', executionContextId); + } + + return `${mvtUrlServicePath}?${params.toString()}`; } async getTimesliceMaskFieldName(): Promise { @@ -935,7 +960,10 @@ export class ESSearchSource extends AbstractESSource implements IMvtVectorSource abortSignal: abortController.signal, sessionId: searchFilters.searchSessionId, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_search_source:all_doc_counts'), + executionContext: mergeExecutionContext( + { description: 'es_search_source:all_doc_counts' }, + searchFilters.executionContext + ), }) ); return !isTotalHitsGreaterThan(resp.hits.total as unknown as TotalHits, maxResultWindow); diff --git a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts index 32d60cac8c7d6..f941c610962d2 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_source/es_source.ts @@ -41,7 +41,7 @@ import { IDynamicStyleProperty } from '../../styles/vector/properties/dynamic_st import { IField } from '../../fields/field'; import { FieldFormatter } from '../../../../common/constants'; import { isValidStringConfig } from '../../util/valid_string_config'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; export function isSearchSourceAbortError(error: Error) { return error.name === 'AbortError'; @@ -62,6 +62,7 @@ export interface IESSource extends IVectorSource { timeFilters, searchSessionId, inspectorAdapters, + executionContext, }: { layerName: string; style: IVectorStyle; @@ -71,6 +72,7 @@ export interface IESSource extends IVectorSource { timeFilters: TimeRange; searchSessionId?: string; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise; } @@ -310,7 +312,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource searchSource.fetch$({ abortSignal: abortController.signal, legacyHitsTotal: false, - executionContext: makePublicExecutionContext('es_source:bounds'), + executionContext: mergeExecutionContext( + { description: 'es_source:bounds' }, + boundsFilters.executionContext + ), }) ); @@ -451,6 +456,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource timeFilters, searchSessionId, inspectorAdapters, + executionContext, }: { layerName: string; style: IVectorStyle; @@ -460,6 +466,7 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource timeFilters: TimeRange; searchSessionId?: string; inspectorAdapters: Adapters; + executionContext: KibanaExecutionContext; }): Promise { const promises = dynamicStyleProps.map((dynamicStyleProp) => { return dynamicStyleProp.getFieldMetaRequest(); @@ -506,7 +513,10 @@ export class AbstractESSource extends AbstractVectorSource implements IESSource } ), searchSessionId, - executionContext: makePublicExecutionContext('es_source:style_meta'), + executionContext: mergeExecutionContext( + { description: 'es_source:style_meta' }, + executionContext + ), requestsAdapter: inspectorAdapters.requests, }); diff --git a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts index a8a756eeb3aed..ce4f70c635612 100644 --- a/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/es_term_source/es_term_source.ts @@ -32,7 +32,7 @@ import { PropertiesMap } from '../../../../common/elasticsearch_util'; import { isValidStringConfig } from '../../util/valid_string_config'; import { ITermJoinSource } from '../term_join_source'; import { IField } from '../../fields/field'; -import { makePublicExecutionContext } from '../../../util'; +import { mergeExecutionContext } from '../execution_context_utils'; const TERMS_AGG_NAME = 'join'; const TERMS_BUCKET_KEYS_TO_IGNORE = ['key', 'doc_count']; @@ -159,7 +159,10 @@ export class ESTermSource extends AbstractESAggSource implements ITermJoinSource }, }), searchSessionId: searchFilters.searchSessionId, - executionContext: makePublicExecutionContext('es_term_source:terms'), + executionContext: mergeExecutionContext( + { description: 'es_term_source:terms' }, + searchFilters.executionContext + ), requestsAdapter: inspectorAdapters.requests, }); diff --git a/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts new file mode 100644 index 0000000000000..5fc9143379208 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { APP_ID } from '../../../common/constants'; +import { getExecutionContextId, mergeExecutionContext } from './execution_context_utils'; + +describe('mergeExecutionContext', () => { + test('should merge with context', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { name: APP_ID }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + description: 'es_pew_pew_source:connections', + name: APP_ID, + }); + }); + + test('should merge with hierarchical context', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { + name: 'dashboard', + child: { + name: APP_ID, + }, + }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + name: 'dashboard', + child: { + description: 'es_pew_pew_source:connections', + name: APP_ID, + }, + }); + }); + + test('should not merge if "maps" context can not be found', () => { + const mergeContext = { description: 'es_pew_pew_source:connections' }; + const context = { + name: 'dashboard', + child: { + name: 'lens', + }, + }; + expect(mergeExecutionContext(mergeContext, context)).toEqual({ + name: 'dashboard', + child: { + name: 'lens', + }, + }); + }); +}); + +describe('getExecutionContextId', () => { + test('should return executionContextId', () => { + const context = { name: APP_ID, id: 'map1234' }; + expect(getExecutionContextId(context)).toBe('map1234'); + }); + + test('should return executionContextId with hierarchical context', () => { + const context = { + name: 'dashboard', + child: { + name: APP_ID, + id: 'map1234', + }, + }; + expect(getExecutionContextId(context)).toBe('map1234'); + }); + + test('should return undefined if "maps" context can not be found', () => { + const context = { + name: 'dashboard', + child: { + name: 'lens', + id: 'lens1234', + }, + }; + expect(getExecutionContextId(context)).toBeUndefined(); + }); +}); diff --git a/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts new file mode 100644 index 0000000000000..4fda822e41994 --- /dev/null +++ b/x-pack/plugins/maps/public/classes/sources/execution_context_utils.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { KibanaExecutionContext } from '@kbn/core/public'; +import { APP_ID } from '../../../common/constants'; + +export function mergeExecutionContext( + mergeContext: Partial, + context: KibanaExecutionContext = {} +): KibanaExecutionContext { + if (isMapContext(context)) { + return { + ...context, + ...mergeContext, + }; + } + + if (context.child !== undefined) { + return { + ...context, + child: { + ...mergeExecutionContext(mergeContext, context.child), + }, + }; + } + + return context; +} + +export function getExecutionContextId(context: KibanaExecutionContext = {}): string | undefined { + if (isMapContext(context)) { + return context.id; + } + + if (context.child !== undefined) { + return getExecutionContextId(context.child); + } + + return undefined; +} + +function isMapContext(context: KibanaExecutionContext): boolean { + return context.name === APP_ID; +} diff --git a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx index 539503b1ebafb..a76d0268c8363 100644 --- a/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/mvt_single_layer_vector_source/mvt_single_layer_vector_source.tsx @@ -208,10 +208,7 @@ export class MVTSingleLayerVectorSource extends AbstractSource implements IMvtVe return []; } - async getTooltipProperties( - properties: GeoJsonProperties, - featureId?: string | number - ): Promise { + async getTooltipProperties(properties: GeoJsonProperties): Promise { const tooltips = []; for (const key in properties) { if (properties.hasOwnProperty(key)) { diff --git a/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts b/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts index 9228fe1de4496..6e3b12be53aaf 100644 --- a/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts +++ b/x-pack/plugins/maps/public/classes/sources/term_join_source/term_join_source.ts @@ -6,6 +6,7 @@ */ import { GeoJsonProperties } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Query } from '@kbn/data-plugin/common/query'; import { Adapters } from '@kbn/inspector-plugin/common/adapters'; import { IField } from '../../fields/field'; @@ -34,6 +35,9 @@ export interface ITermJoinSource extends ISource { getId(): string; getRightFields(): IField[]; - getTooltipProperties(properties: GeoJsonProperties): Promise; + getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; getFieldByName(fieldName: string): IField | null; } diff --git a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx index 0620386fda4c1..020b99fc429e2 100644 --- a/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx +++ b/x-pack/plugins/maps/public/classes/sources/vector_source/vector_source.tsx @@ -14,6 +14,7 @@ import { Polygon, Position, } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import type { Query } from '@kbn/data-plugin/common'; import type { MapGeoJSONFeature } from '@kbn/mapbox-gl'; @@ -62,6 +63,7 @@ export interface BoundsRequestMeta { timeslice?: Timeslice; isFeatureEditorOpenForLayer: boolean; joinKeyFilter?: Filter; + executionContext: KibanaExecutionContext; } export interface GetFeatureActionsArgs { @@ -84,7 +86,10 @@ export interface GetFeatureActionsArgs { export interface IVectorSource extends ISource { isMvt(): boolean; - getTooltipProperties(properties: GeoJsonProperties): Promise; + getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise; getBoundsForFilters( layerDataFilters: BoundsRequestMeta, registerCancelCallback: (callback: () => void) => void @@ -202,7 +207,10 @@ export class AbstractVectorSource extends AbstractSource implements IVectorSourc } // Allow source to filter and format feature properties before displaying to user - async getTooltipProperties(properties: GeoJsonProperties): Promise { + async getTooltipProperties( + properties: GeoJsonProperties, + executionContext: KibanaExecutionContext + ): Promise { const tooltipProperties: ITooltipProperty[] = []; for (const key in properties) { if (key.startsWith('__kbn')) { diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap index c5784cf9a309a..de542ccfc8e8e 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/__snapshots__/tooltip_control.test.tsx.snap @@ -6,6 +6,7 @@ exports[`TooltipControl render should render hover tooltip 1`] = ` void; + executionContext: KibanaExecutionContext; } export class TooltipControl extends Component { @@ -376,6 +378,7 @@ export class TooltipControl extends Component { isLocked={isLocked} index={index} loadFeatureGeometry={this._getFeatureGeometry} + executionContext={this.props.executionContext} /> ); }); diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx index cb96f364925b4..849f052827ebe 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.test.tsx @@ -84,6 +84,7 @@ const defaultProps = { loadFeatureGeometry: () => { return null; }, + executionContext: {}, }; describe('TooltipPopover', () => { diff --git a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx index 5c1f8ef6a83a1..0263c523f0fce 100644 --- a/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx +++ b/x-pack/plugins/maps/public/connected_components/mb_map/tooltip_control/tooltip_popover.tsx @@ -9,6 +9,7 @@ import React, { Component, RefObject } from 'react'; import { EuiPopover, EuiText } from '@elastic/eui'; import type { Map as MbMap } from '@kbn/mapbox-gl'; import { GeoJsonProperties, Geometry } from 'geojson'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import { ActionExecutionContext, Action } from '@kbn/ui-actions-plugin/public'; import { FeaturesTooltip } from './features_tooltip'; @@ -39,6 +40,7 @@ interface Props { mbMap: MbMap; onSingleValueTrigger?: (actionId: string, key: string, value: RawValue) => void; renderTooltipContent?: RenderToolTipContent; + executionContext: KibanaExecutionContext; } interface State { @@ -98,7 +100,7 @@ export class TooltipPopover extends Component { return []; } - return await tooltipLayer.getPropertiesForTooltip(properties); + return await tooltipLayer.getPropertiesForTooltip(properties, this.props.executionContext); }; _getLayerName = async (layerId: string) => { diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx b/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx index bcb5aea3cca85..1528290dcde2f 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable.test.tsx @@ -15,6 +15,13 @@ import { MapSavedObjectAttributes } from '../../common/map_saved_object_type'; jest.mock('../kibana_services', () => { return { + getExecutionContextService() { + return { + get: () => { + return {}; + }, + }; + }, getHttp() { return { basePath: { diff --git a/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx b/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx index c109b4f940d81..3c6eaf77062c3 100644 --- a/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx +++ b/x-pack/plugins/maps/public/embeddable/map_embeddable.tsx @@ -21,6 +21,7 @@ import { startWith, } from 'rxjs/operators'; import { Unsubscribe } from 'redux'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { EuiEmptyPrompt } from '@elastic/eui'; import { type Filter } from '@kbn/es-query'; import { KibanaThemeProvider } from '@kbn/kibana-react-plugin/public'; @@ -46,6 +47,7 @@ import { updateLayerById, setGotoWithCenter, setEmbeddableSearchContext, + setExecutionContext, } from '../actions'; import { getIsLayerTOCOpen, getOpenTOCDetails } from '../selectors/ui_selectors'; import { @@ -80,13 +82,14 @@ import { } from '../../common/constants'; import { RenderToolTipContent } from '../classes/tooltips/tooltip_property'; import { - getUiActions, + getChartsPaletteServiceGetColor, getCoreI18n, + getExecutionContextService, getHttp, - getChartsPaletteServiceGetColor, - getSpacesApi, getSearchService, + getSpacesApi, getTheme, + getUiActions, } from '../kibana_services'; import { LayerDescriptor, MapExtent } from '../../common/descriptor_types'; import { MapContainer } from '../connected_components/map_container'; @@ -194,6 +197,8 @@ export class MapEmbeddable return; } + this._savedMap.getStore().dispatch(setExecutionContext(this.getExecutionContext())); + // deferred loading of this embeddable is complete this.setInitializationFinished(); @@ -203,6 +208,23 @@ export class MapEmbeddable } } + private getExecutionContext() { + const parentContext = getExecutionContextService().get(); + const mapContext: KibanaExecutionContext = { + type: APP_ID, + name: APP_ID, + id: this.id, + url: this.output.editPath, + }; + + return parentContext + ? { + ...parentContext, + child: mapContext, + } + : mapContext; + } + private _initializeStore() { this._dispatchSetChartsPaletteServiceGetColor(this.input.syncColors); diff --git a/x-pack/plugins/maps/public/kibana_services.ts b/x-pack/plugins/maps/public/kibana_services.ts index 0e4ee6d913e32..b9c64946f7fa3 100644 --- a/x-pack/plugins/maps/public/kibana_services.ts +++ b/x-pack/plugins/maps/public/kibana_services.ts @@ -41,7 +41,7 @@ export const getIndexPatternSelectComponent = () => pluginsStart.unifiedSearch.ui.IndexPatternSelect; export const getSearchBar = () => pluginsStart.unifiedSearch.ui.SearchBar; export const getHttp = () => coreStart.http; -export const getExecutionContext = () => coreStart.executionContext; +export const getExecutionContextService = () => coreStart.executionContext; export const getTimeFilter = () => pluginsStart.data.query.timefilter.timefilter; export const getToasts = () => coreStart.notifications.toasts; export const getSavedObjectsClient = () => coreStart.savedObjects.client; diff --git a/x-pack/plugins/maps/public/reducers/map/map.ts b/x-pack/plugins/maps/public/reducers/map/map.ts index 41de8ab0b8b0f..8215cf0fe7f9f 100644 --- a/x-pack/plugins/maps/public/reducers/map/map.ts +++ b/x-pack/plugins/maps/public/reducers/map/map.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { APP_ID } from '../../../common/constants'; import { SET_SELECTED_LAYER, UPDATE_LAYER_ORDER, @@ -46,6 +47,7 @@ import { TRACK_MAP_SETTINGS, UPDATE_MAP_SETTING, UPDATE_EDIT_STATE, + SET_EXECUTION_CONTEXT, } from '../../actions/map_action_constants'; import { getDefaultMapSettings } from './default_map_settings'; @@ -63,6 +65,7 @@ import { startDataRequest, stopDataRequest, updateSourceDataRequest } from './da import { MapState } from './types'; export const DEFAULT_MAP_STATE: MapState = { + executionContext: { name: APP_ID }, ready: false, mapInitError: null, goto: null, @@ -322,6 +325,12 @@ export function map(state: MapState = DEFAULT_MAP_STATE, action: Record & { }; export type MapState = { + executionContext: KibanaExecutionContext; ready: boolean; mapInitError?: string | null; goto?: Goto | null; diff --git a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx index 3dab6db00885d..759189f509db0 100644 --- a/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx +++ b/x-pack/plugins/maps/public/routes/list_page/maps_list_view.tsx @@ -17,7 +17,7 @@ import { APP_ID, getEditPath, MAP_PATH, MAP_SAVED_OBJECT_TYPE } from '../../../c import { getMapsCapabilities, getCoreChrome, - getExecutionContext, + getExecutionContextService, getNavigateToApp, getSavedObjectsClient, getUiSettings, @@ -104,10 +104,10 @@ interface Props { } export function MapsListView(props: Props) { - getExecutionContext().set({ + getExecutionContextService().set({ type: 'application', + name: APP_ID, page: 'list', - id: '', }); const isReadOnly = !getMapsCapabilities().save; diff --git a/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts b/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts index af9054cd99570..649c5033fb628 100644 --- a/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts +++ b/x-pack/plugins/maps/public/routes/map_page/map_app/index.ts @@ -8,6 +8,7 @@ import { connect } from 'react-redux'; import { ThunkDispatch } from 'redux-thunk'; import { AnyAction } from 'redux'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import { Filter } from '@kbn/es-query'; import type { Query, TimeRange } from '@kbn/es-query'; import { MapApp } from './map_app'; @@ -19,7 +20,7 @@ import { getTimeFilters, hasDirtyState, } from '../../../selectors/map_selectors'; -import { setQuery, enableFullScreen, openMapSettings } from '../../../actions'; +import { setQuery, setExecutionContext, enableFullScreen, openMapSettings } from '../../../actions'; import { FLYOUT_STATE } from '../../../reducers/ui'; import { getInspectorAdapters } from '../../../reducers/non_serializable_instances'; import { MapStoreState } from '../../../reducers/store'; @@ -65,6 +66,8 @@ function mapDispatchToProps(dispatch: ThunkDispatch dispatch(enableFullScreen()), openMapSettings: () => dispatch(openMapSettings()), + setExecutionContext: (executionContext: KibanaExecutionContext) => + dispatch(setExecutionContext(executionContext)), }; } diff --git a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx index bf8963de0461d..969334551adfe 100644 --- a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx +++ b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx @@ -9,7 +9,12 @@ import React from 'react'; import _ from 'lodash'; import { finalize, switchMap, tap } from 'rxjs/operators'; import { i18n } from '@kbn/i18n'; -import { AppLeaveAction, AppMountParameters, ScopedHistory } from '@kbn/core/public'; +import { + AppLeaveAction, + AppMountParameters, + KibanaExecutionContext, + ScopedHistory, +} from '@kbn/core/public'; import { Adapters } from '@kbn/embeddable-plugin/public'; import { Subscription } from 'rxjs'; import { type Filter, FilterStateStore, type Query, type TimeRange } from '@kbn/es-query'; @@ -29,7 +34,7 @@ import { } from '@kbn/kibana-utils-plugin/public'; import { getData, - getExecutionContext, + getExecutionContextService, getCoreChrome, getIndexPatternService, getMapsCapabilities, @@ -84,6 +89,7 @@ export interface Props { query: Query | undefined; setHeaderActionMenu: AppMountParameters['setHeaderActionMenu']; history: ScopedHistory; + setExecutionContext: (executionContext: KibanaExecutionContext) => void; } export interface State { @@ -123,11 +129,15 @@ export class MapApp extends React.Component { componentDidMount() { this._isMounted = true; - getExecutionContext().set({ + const executionContext = { type: 'application', - page: 'editor', + name: APP_ID, + url: window.location.pathname, id: this.props.savedMap.getSavedObjectId() || 'new', - }); + page: 'editor', + }; + getExecutionContextService().set(executionContext); // set execution context in core ExecutionContextStartService + this.props.setExecutionContext(executionContext); // set execution context in redux store this._autoRefreshSubscription = getTimeFilter() .getAutoRefreshFetch$() diff --git a/x-pack/plugins/maps/public/selectors/map_selectors.test.ts b/x-pack/plugins/maps/public/selectors/map_selectors.test.ts index dcd6a727238d5..fc051d7a48a1e 100644 --- a/x-pack/plugins/maps/public/selectors/map_selectors.test.ts +++ b/x-pack/plugins/maps/public/selectors/map_selectors.test.ts @@ -71,6 +71,7 @@ describe('getDataFilters', () => { minLon: -0.25, }; const isReadOnly = false; + const executionContext = {}; test('should set buffer as searchSessionMapBuffer when using searchSessionId', () => { const dataFilters = getDataFilters.resultFunc( @@ -84,7 +85,8 @@ describe('getDataFilters', () => { embeddableSearchContext, searchSessionId, searchSessionMapBuffer, - isReadOnly + isReadOnly, + executionContext ); expect(dataFilters.buffer).toEqual(searchSessionMapBuffer); }); @@ -101,7 +103,8 @@ describe('getDataFilters', () => { embeddableSearchContext, searchSessionId, undefined, - isReadOnly + isReadOnly, + executionContext ); expect(dataFilters.buffer).toEqual(mapBuffer); }); diff --git a/x-pack/plugins/maps/public/selectors/map_selectors.ts b/x-pack/plugins/maps/public/selectors/map_selectors.ts index e8aae1c61dda1..798e24a6141e1 100644 --- a/x-pack/plugins/maps/public/selectors/map_selectors.ts +++ b/x-pack/plugins/maps/public/selectors/map_selectors.ts @@ -8,6 +8,7 @@ import { createSelector } from 'reselect'; import { FeatureCollection } from 'geojson'; import _ from 'lodash'; +import type { KibanaExecutionContext } from '@kbn/core/public'; import type { Query } from '@kbn/data-plugin/common'; import { Filter } from '@kbn/es-query'; import type { TimeRange } from '@kbn/es-query'; @@ -239,6 +240,10 @@ export function getDataRequestDescriptor(state: MapStoreState, layerId: string, }); } +export function getExecutionContext(state: MapStoreState): KibanaExecutionContext { + return state.map.executionContext; +} + export const getDataFilters = createSelector( getMapExtent, getMapBuffer, @@ -251,6 +256,7 @@ export const getDataFilters = createSelector( getSearchSessionId, getSearchSessionMapBuffer, getIsReadOnly, + getExecutionContext, ( mapExtent, mapBuffer, @@ -262,7 +268,8 @@ export const getDataFilters = createSelector( embeddableSearchContext, searchSessionId, searchSessionMapBuffer, - isReadOnly + isReadOnly, + executionContext ) => { return { extent: mapExtent, @@ -275,6 +282,7 @@ export const getDataFilters = createSelector( embeddableSearchContext, searchSessionId, isReadOnly, + executionContext, }; } ); diff --git a/x-pack/plugins/maps/public/util.test.js b/x-pack/plugins/maps/public/util.test.js deleted file mode 100644 index f07d60fc3d702..0000000000000 --- a/x-pack/plugins/maps/public/util.test.js +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { makePublicExecutionContext } from './util'; - -describe('makePublicExecutionContext', () => { - let injectedContext = {}; - beforeAll(() => { - require('./kibana_services').getExecutionContext = () => ({ - get: () => injectedContext, - }); - }); - - test('creates basic context when no top level context is provided', () => { - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - description: 'test', - name: 'maps', - type: 'application', - url: '/', - }); - }); - - test('merges with top level context if its from the same app', () => { - injectedContext = { - name: 'maps', - id: '1234', - }; - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - description: 'test', - name: 'maps', - type: 'application', - url: '/', - id: '1234', - }); - }); - - test('nests inside top level context if its from a different app', () => { - injectedContext = { - name: 'other-app', - id: '1234', - }; - const context = makePublicExecutionContext('test'); - expect(context).toStrictEqual({ - name: 'other-app', - id: '1234', - child: { - description: 'test', - type: 'application', - name: 'maps', - url: '/', - }, - }); - }); -}); diff --git a/x-pack/plugins/maps/public/util.ts b/x-pack/plugins/maps/public/util.ts index 68a616f870000..364f72c0b564e 100644 --- a/x-pack/plugins/maps/public/util.ts +++ b/x-pack/plugins/maps/public/util.ts @@ -6,15 +6,8 @@ */ import { EMSClient, FileLayer, TMSService } from '@elastic/ems-client'; -import type { KibanaExecutionContext } from '@kbn/core/public'; -import { - getTilemap, - getEMSSettings, - getMapsEmsStart, - getExecutionContext, -} from './kibana_services'; +import { getTilemap, getEMSSettings, getMapsEmsStart } from './kibana_services'; import { getLicenseId } from './licensed_features'; -import { makeExecutionContext } from '../common/execution_context'; export function getKibanaTileMap(): unknown { return getTilemap(); @@ -61,22 +54,3 @@ async function getEMSClient(): Promise { export function isRetina(): boolean { return window.devicePixelRatio === 2; } - -export function makePublicExecutionContext(description: string): KibanaExecutionContext { - const topLevelContext = getExecutionContext().get(); - const context = makeExecutionContext({ - url: window.location.pathname, - description, - }); - - // Distinguish between running in maps app vs. embedded - return topLevelContext.name !== undefined && topLevelContext.name !== context.name - ? { - ...topLevelContext, - child: context, - } - : { - ...topLevelContext, - ...context, - }; -} diff --git a/x-pack/plugins/maps/server/mvt/mvt_routes.ts b/x-pack/plugins/maps/server/mvt/mvt_routes.ts index 4ed26677b6286..0b2e8c51d1eb8 100644 --- a/x-pack/plugins/maps/server/mvt/mvt_routes.ts +++ b/x-pack/plugins/maps/server/mvt/mvt_routes.ts @@ -15,12 +15,12 @@ import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import { errors } from '@elastic/elasticsearch'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { + APP_ID, MVT_GETTILE_API_PATH, API_ROOT_PATH, MVT_GETGRIDTILE_API_PATH, RENDER_AS, } from '../../common/constants'; -import { makeExecutionContext } from '../../common/execution_context'; import { getAggsTileRequest, getHitsTileRequest } from '../../common/mvt_request_body'; const CACHE_TIMEOUT_SECONDS = 60 * 60; @@ -50,6 +50,7 @@ export function initMVTRoutes({ requestBody: schema.string(), index: schema.string(), token: schema.maybe(schema.string()), + executionContextId: schema.maybe(schema.string()), }), }, }, @@ -88,8 +89,11 @@ export function initMVTRoutes({ context, core, executionContext: makeExecutionContext({ + type: 'server', + name: APP_ID, description: 'mvt:get_hits_tile', url: `${API_ROOT_PATH}/${MVT_GETTILE_API_PATH}/${z}/${x}/${y}.pbf`, + id: query.executionContextId, }), logger, path: tileRequest.path, @@ -117,6 +121,7 @@ export function initMVTRoutes({ renderAs: schema.string(), token: schema.maybe(schema.string()), gridPrecision: schema.number(), + executionContextId: schema.maybe(schema.string()), }), }, }, @@ -157,8 +162,11 @@ export function initMVTRoutes({ context, core, executionContext: makeExecutionContext({ + type: 'server', + name: APP_ID, description: 'mvt:get_aggs_tile', url: `${API_ROOT_PATH}/${MVT_GETGRIDTILE_API_PATH}/${z}/${x}/${y}.pbf`, + id: query.executionContextId, }), logger, path: tileRequest.path, @@ -270,3 +278,32 @@ function makeAbortController( }); return abortController; } + +function makeExecutionContext({ + type, + name, + description, + url, + id, +}: { + type: string; + name: string; + description: string; + url: string; + id?: string; +}): KibanaExecutionContext { + return id !== undefined + ? { + type, + name, + description, + url, + id, + } + : { + type, + name, + description, + url, + }; +} diff --git a/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx b/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx index 2d95f0c971696..74f4bf66a0e08 100644 --- a/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx +++ b/x-pack/plugins/ml/public/application/components/ml_page/notifications_indicator.tsx @@ -76,6 +76,7 @@ export const NotificationsIndicator: FC = () => { = ({ memoryOverview }), }, ]} - marker={} + marker={ + + } markerPosition={Position.Top} /> diff --git a/x-pack/test/accessibility/config.ts b/x-pack/test/accessibility/config.ts index 191a8b403c41b..140d6fe6de108 100644 --- a/x-pack/test/accessibility/config.ts +++ b/x-pack/test/accessibility/config.ts @@ -23,7 +23,8 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./apps/grok_debugger'), require.resolve('./apps/search_profiler'), require.resolve('./apps/painless_lab'), - require.resolve('./apps/uptime'), + // https://github.com/elastic/kibana/issues/153601 + // require.resolve('./apps/uptime'), require.resolve('./apps/spaces'), require.resolve('./apps/advanced_settings'), require.resolve('./apps/dashboard_panel_options'), @@ -32,8 +33,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { require.resolve('./apps/roles'), require.resolve('./apps/ingest_node_pipelines'), require.resolve('./apps/index_lifecycle_management'), - require.resolve('./apps/ml'), - require.resolve('./apps/transform'), + // https://github.com/elastic/kibana/issues/153596 + // https://github.com/elastic/kibana/issues/153592 + // require.resolve('./apps/ml'), + // require.resolve('./apps/transform'), require.resolve('./apps/lens'), require.resolve('./apps/upgrade_assistant'), require.resolve('./apps/canvas'), @@ -45,8 +48,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { // Please make sure that the remote clusters, snapshot and restore and // CCR tests stay in that order. Their execution fails if rearranged. require.resolve('./apps/remote_clusters'), - require.resolve('./apps/snapshot_and_restore'), - require.resolve('./apps/cross_cluster_replication'), + // https://github.com/elastic/kibana/issues/153788 + // require.resolve('./apps/snapshot_and_restore'), + // https://github.com/elastic/kibana/issues/153599 + // require.resolve('./apps/cross_cluster_replication'), require.resolve('./apps/reporting'), require.resolve('./apps/enterprise_search'), require.resolve('./apps/license_management'), diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts index 89ba33449ec16..ee8f851489e7b 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts @@ -74,9 +74,32 @@ export default function ({ getService }: FtrProviderContext) { expect(res.cspm.status).to.be('not-deployed'); expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-installed'); expect(res.cspm.healthyAgents).to.be(0); expect(res.cspm.installedPackagePolicies).to.be(1); }); + + it(`Should return not-deployed when vuln_mgmt is not installed`, async () => { + await createPackagePolicy( + supertest, + agentPolicyId, + 'vuln_mgmt', + 'cloudbeat/vuln_mgmt_aws', + 'aws', + 'vuln_mgmt' + ); + + const { body: res }: { body: CspSetupStatus } = await supertest + .get(`/internal/cloud_security_posture/status`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + expect(res.cspm.status).to.be('not-installed'); + expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-deployed'); + expect(res.vuln_mgmt.healthyAgents).to.be(0); + expect(res.vuln_mgmt.installedPackagePolicies).to.be(1); + }); }); } @@ -88,6 +111,26 @@ async function createPackagePolicy( deployment: string, posture: string ) { + const version = posture === 'kspm' || posture === 'cspm' ? '1.2.8' : '1.3.0-preview2'; + const title = 'Security Posture Management'; + const streams = [ + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'cloud_security_posture.vulnerabilities', + }, + }, + ]; + + const inputTemplate = { + enabled: true, + type: input, + policy_template: policyTemplate, + }; + + const inputs = posture === 'vuln_mgmt' ? { ...inputTemplate, streams } : { ...inputTemplate }; + const { body: postPackageResponse } = await supertest .post(`/api/fleet/package_policies`) .set('kbn-xsrf', 'xxxx') @@ -98,17 +141,11 @@ async function createPackagePolicy( namespace: 'default', policy_id: agentPolicyId, enabled: true, - inputs: [ - { - enabled: true, - type: input, - policy_template: policyTemplate, - }, - ], + inputs: [inputs], package: { name: 'cloud_security_posture', - title: 'Kubernetes Security Posture Management', - version: '1.2.8', + title, + version, }, vars: { deployment: { diff --git a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js index c8f1835a11647..d4057948253f8 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_geotile_grid.js @@ -44,13 +44,14 @@ export default function ({ getPageObjects, getService }) { delete searchParams.token; expect(searchParams).to.eql({ - buffer: 4, + buffer: '4', + executionContextId: '78116c8c-fd2a-11ea-adc1-0242ac120002', geometryFieldName: 'geo.coordinates', hasLabels: 'false', index: 'logstash-*', - gridPrecision: 8, + gridPrecision: '8', renderAs: 'grid', - requestBody: `(_source:(excludes:!()),aggs:(max_of_bytes:(max:(field:bytes))),fields:!((field:'@timestamp',format:date_time),(field:'relatedContent.article:modified_time',format:date_time),(field:'relatedContent.article:published_time',format:date_time),(field:utc_time,format:date_time)),query:(bool:(filter:!((range:('@timestamp':(format:strict_date_optional_time,gte:'2015-09-20T00:00:00.000Z',lte:'2015-09-20T01:00:00.000Z')))),must:!(),must_not:!(),should:!())),runtime_mappings:(),script_fields:(hour_of_day:(script:(lang:painless,source:'doc[!'@timestamp!'].value.getHour()'))),size:0,stored_fields:!('*'))`, + requestBody: `(_source%3A(excludes%3A!())%2Caggs%3A(max_of_bytes%3A(max%3A(field%3Abytes)))%2Cfields%3A!((field%3A'%40timestamp'%2Cformat%3Adate_time)%2C(field%3A'relatedContent.article%3Amodified_time'%2Cformat%3Adate_time)%2C(field%3A'relatedContent.article%3Apublished_time'%2Cformat%3Adate_time)%2C(field%3Autc_time%2Cformat%3Adate_time))%2Cquery%3A(bool%3A(filter%3A!((range%3A('%40timestamp'%3A(format%3Astrict_date_optional_time%2Cgte%3A'2015-09-20T00%3A00%3A00.000Z'%2Clte%3A'2015-09-20T01%3A00%3A00.000Z'))))%2Cmust%3A!()%2Cmust_not%3A!()%2Cshould%3A!()))%2Cruntime_mappings%3A()%2Cscript_fields%3A(hour_of_day%3A(script%3A(lang%3Apainless%2Csource%3A'doc%5B!'%40timestamp!'%5D.value.getHour()')))%2Csize%3A0%2Cstored_fields%3A!('*'))`, }); //Should correctly load meta for style-rule (sigma is set to 1, opacity to 1) diff --git a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js index 80a7f8abc5d36..744547cfaf5e2 100644 --- a/x-pack/test/functional/apps/maps/group4/mvt_scaling.js +++ b/x-pack/test/functional/apps/maps/group4/mvt_scaling.js @@ -49,12 +49,13 @@ export default function ({ getPageObjects, getService }) { delete searchParams.token; expect(searchParams).to.eql({ - buffer: 4, + buffer: '4', + executionContextId: 'bff99716-e3dc-11ea-87d0-0242ac130003', geometryFieldName: 'geometry', hasLabels: 'false', index: 'geo_shapes*', requestBody: - '(_source:!f,fields:!(prop1),query:(bool:(filter:!(),must:!(),must_not:!(),should:!())),runtime_mappings:(),size:10001)', + '(_source%3A!f%2Cfields%3A!(prop1)%2Cquery%3A(bool%3A(filter%3A!()%2Cmust%3A!()%2Cmust_not%3A!()%2Cshould%3A!()))%2Cruntime_mappings%3A()%2Csize%3A10001)', }); }); diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts index d7560a9533f3d..ac3be7d1814a5 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_alerting/list_view.ts @@ -11,37 +11,56 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const commonScreenshots = getService('commonScreenshots'); const screenshotDirectories = ['response_ops_docs', 'stack_alerting']; const pageObjects = getPageObjects(['common', 'header']); + const actions = getService('actions'); const rules = getService('rules'); const testSubjects = getService('testSubjects'); + const ruleName = 'kibana sites - low bytes'; describe('list view', function () { let ruleId: string; - const indexThresholdRule = { - consumer: 'alerts', - name: 'my rule', - notifyWhen: 'onActionGroupChange', - params: { - index: ['.test-index'], - timeField: '@timestamp', - aggType: 'count', - groupBy: 'all', - timeWindowSize: 5, - timeWindowUnit: 'd', - thresholdComparator: '>', - threshold: [1000], - }, - ruleTypeId: '.index-threshold', - schedule: { interval: '1m' }, - tags: [], - actions: [], - }; - + let connectorId: string; before(async () => { - ({ id: ruleId } = await rules.api.createRule(indexThresholdRule)); + ({ id: connectorId } = await actions.api.createConnector({ + name: 'my-server-log-connector', + config: {}, + secrets: {}, + connectorTypeId: '.server-log', + })); + ({ id: ruleId } = await rules.api.createRule({ + consumer: 'alerts', + name: ruleName, + notifyWhen: 'onActionGroupChange', + params: { + index: ['kibana_sample_data_logs'], + timeField: '@timestamp', + aggType: 'sum', + aggField: 'bytes', + groupBy: 'top', + termField: 'host.keyword', + termSize: 4, + timeWindowSize: 24, + timeWindowUnit: 'h', + thresholdComparator: '>', + threshold: [4200], + }, + ruleTypeId: '.index-threshold', + schedule: { interval: '1m' }, + actions: [ + { + group: 'threshold met', + id: connectorId, + params: { + level: 'info', + message: 'Test', + }, + }, + ], + })); }); after(async () => { await rules.api.deleteRule(ruleId); + await actions.api.deleteConnector(connectorId); }); it('rules list screenshot', async () => { @@ -71,5 +90,29 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await snoozeBadge.click(); await commonScreenshots.takeScreenshot('snooze-panel', screenshotDirectories, 1400, 1024); }); + + it('rule detail screenshots', async () => { + await pageObjects.common.navigateToApp('triggersActions'); + await pageObjects.header.waitUntilLoadingHasFinished(); + await testSubjects.setValue('ruleSearchField', ruleName); + const rulesList = await testSubjects.find('rulesList'); + const alertRule = await rulesList.findByCssSelector(`[title="${ruleName}"]`); + await alertRule.click(); + await pageObjects.header.waitUntilLoadingHasFinished(); + await commonScreenshots.takeScreenshot( + 'rule-details-alerts-active', + screenshotDirectories, + 1400, + 1024 + ); + const actionsButton = await testSubjects.find('ruleActionsButton'); + await actionsButton.click(); + await commonScreenshots.takeScreenshot( + 'rule-details-disabling', + screenshotDirectories, + 1400, + 1024 + ); + }); }); } diff --git a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts index aa5ac66aa9e2a..247b25d2be2ca 100644 --- a/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts +++ b/x-pack/test/screenshot_creation/apps/response_ops_docs/stack_connectors/connector_types.ts @@ -14,7 +14,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const actions = getService('actions'); const testSubjects = getService('testSubjects'); const comboBox = getService('comboBox'); - const es = getService('es'); const testIndex = `test-index`; const indexDocument = `{\n` + @@ -43,19 +42,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('index connector screenshots', async () => { - await es.indices.create({ - index: testIndex, - body: { - mappings: { - properties: { - date_updated: { - type: 'date', - format: 'epoch_millis', - }, - }, - }, - }, - }); await pageObjects.common.navigateToApp('connectors'); await pageObjects.header.waitUntilLoadingHasFinished(); await actions.common.openNewConnectorForm('index'); @@ -71,8 +57,5 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { const flyOutCancelButton = await testSubjects.find('euiFlyoutCloseButton'); await flyOutCancelButton.click(); }); - after(async () => { - await es.indices.delete({ index: testIndex }); - }); }); } diff --git a/yarn.lock b/yarn.lock index 69d01d1874077..3b8ee9318c67b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10657,6 +10657,11 @@ axe-core@^4.2.0: resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.3.5.tgz#78d6911ba317a8262bfee292aeafcc1e04b49cc5" integrity sha512-WKTW1+xAzhMS5dJsxWkliixlO/PqC4VhmO9T4juNYcaTg9jzWiJsou6m5pxWYGfigWbwzJWeFY6z47a+4neRXA== +axe-core@^4.6.1: + version "4.6.1" + resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.6.1.tgz#79cccdee3e3ab61a8f42c458d4123a6768e6fbce" + integrity sha512-lCZN5XRuOnpG4bpMq8v0khrWtUOn+i8lZSb6wHZH56ZfbIEv6XwJV84AAueh9/zi7qPVJ/E4yz6fmsiyOmXR4w== + axios@^0.21.1: version "0.21.4" resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575"