From 51244c86f75808468928e8ad8d55b6a74551876b Mon Sep 17 00:00:00 2001 From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 15 Aug 2021 11:21:57 -0400 Subject: [PATCH] [RAC][Rule Registry] Improve RuleDataService API and index bootstrapping implementation (#108115) (#108638) **Addresses:** https://github.com/elastic/kibana/issues/106421, https://github.com/elastic/kibana/issues/106428, https://github.com/elastic/kibana/issues/102089, https://github.com/elastic/kibana/issues/106433 ## Summary This PR focuses on consolidation of indexing implementations in `rule_registry` (https://github.com/elastic/kibana/issues/101016). It addresses some of the sub-tasks of the parent ticket. - [x] Encapsulate index bootstrapping logic in a new improved API exposed by `RuleDataService`. - [x] Enforce allowed values for the `datasetSuffix` on the API level. - [x] Migrate plugins using the existing `RuleDataService` API to the improved one. - [x] Make sure index names comply with design architecture. - https://github.com/elastic/kibana/issues/102089 - [x] Improve the API of `RuleDataClient`. - [x] Enhance index bootstrapping: support custom ILM policy per index (`{registrationContext}.{datasetSuffix}`). - [x] Enhance index bootstrapping: create index template per namespace and support rollovers properly - based on https://github.com/elastic/kibana/pull/107700 - [x] Enhance index bootstrapping: support secondary aliases - based on https://github.com/elastic/kibana/pull/107700 - [x] Remove `EventLogService` implementation - https://github.com/elastic/kibana/issues/106433 This will be addressed in follow-up PRs: - [ ] Enhance index bootstrapping: implement suggestions for backwards compatibility (naming scheme for alias and backing indices; versioning). - [ ] Enhance index bootstrapping: implement upgrades of existing index templates. - [ ] Make index bootstrapping logic more robust. This _is partially addressed_ in this PR, but more improvements are needed. - [ ] Change the way index prefix works. - [ ] Add support for optional TS schema (static typing). - [ ] Update `README` in `rule_registry`. ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Georgii Gorbachev --- .../server/lib/alerts/register_apm_alerts.ts | 4 +- .../apm/server/lib/alerts/test_utils/index.ts | 5 +- .../server/lib/services/get_service_alerts.ts | 4 +- x-pack/plugins/apm/server/plugin.ts | 100 ++--- x-pack/plugins/apm/server/routes/typings.ts | 4 +- .../server/services/rules/rule_data_client.ts | 63 +-- .../infra/server/services/rules/types.ts | 4 +- x-pack/plugins/observability/server/plugin.ts | 18 +- .../server/routes/register_routes.ts | 4 +- .../observability/server/routes/types.ts | 4 +- x-pack/plugins/rule_registry/common/types.ts | 4 - x-pack/plugins/rule_registry/kibana.json | 1 - .../server/event_log/elasticsearch/index.ts | 14 - .../elasticsearch/index_bootstrapper.ts | 107 ----- .../elasticsearch/index_management_gateway.ts | 140 ------ .../event_log/elasticsearch/index_reader.ts | 44 -- .../event_log/elasticsearch/index_writer.ts | 94 ---- .../elasticsearch/resources/ilm_policy.ts | 33 -- .../elasticsearch/resources/index_mappings.ts | 12 - .../elasticsearch/resources/index_names.ts | 84 ---- .../elasticsearch/resources/index_template.ts | 52 --- .../elasticsearch/utils/buffered_stream.ts | 52 --- .../server/event_log/event_schema/index.ts | 9 - .../server/event_log/event_schema/schema.ts | 51 --- .../event_log/event_schema/schema_types.ts | 20 - .../rule_registry/server/event_log/index.ts | 10 - .../server/event_log/log/event_log.ts | 51 --- .../event_log/log/event_log_bootstrapper.ts | 51 --- .../event_log/log/event_log_definition.ts | 37 -- .../event_log/log/event_log_provider.ts | 33 -- .../event_log/log/event_log_registry.ts | 56 --- .../event_log/log/event_log_resolver.ts | 162 ------- .../server/event_log/log/event_log_service.ts | 67 --- .../server/event_log/log/event_logger.ts | 36 -- .../event_log/log/event_logger_template.ts | 55 --- .../server/event_log/log/event_query.ts | 27 -- .../event_log/log/event_query_builder.ts | 110 ----- .../server/event_log/log/index.ts | 10 - .../server/event_log/log/internal_api.ts | 46 -- .../server/event_log/log/public_api.ts | 113 ----- .../log/utils/mapping_from_field_map.ts | 36 -- .../server/event_log/utils/fields.ts | 18 - .../server/event_log/utils/predicates.ts | 16 - .../event_log/utils/ready_signal.test.ts | 22 - .../server/event_log/utils/ready_signal.ts | 29 -- .../server/event_log/utils/utility_types.ts | 12 - x-pack/plugins/rule_registry/server/index.ts | 16 +- x-pack/plugins/rule_registry/server/plugin.ts | 61 +-- .../server/rule_data_client/index.ts | 177 +------- ...lient_mock.ts => rule_data_client.mock.ts} | 20 +- .../rule_data_client/rule_data_client.ts | 153 +++++++ .../server/rule_data_client/types.ts | 38 +- .../server/rule_data_plugin_service/index.ts | 243 +--------- .../rule_data_plugin_service/index_info.ts | 135 ++++++ .../rule_data_plugin_service/index_options.ts | 153 +++++++ .../resource_installer.ts | 421 ++++++++++++++++++ .../rule_data_plugin_service.mock.ts | 15 +- .../rule_data_plugin_service.ts | 151 +++++++ .../server/rule_data_plugin_service/utils.ts | 12 +- .../utils/create_lifecycle_executor.test.ts | 2 +- .../server/utils/create_lifecycle_executor.ts | 4 +- .../utils/create_lifecycle_rule_type.test.ts | 2 +- .../create_lifecycle_rule_type_factory.ts | 4 +- .../server/utils/persistence_types.ts | 4 +- .../utils/with_rule_data_client_factory.ts | 8 +- x-pack/plugins/rule_registry/tsconfig.json | 1 - .../routes/index/create_index_route.ts | 2 +- .../routes/rules/create_rules_route.ts | 4 +- .../routes/rules/delete_rules_route.ts | 4 +- .../routes/rules/find_rules_route.ts | 4 +- .../routes/rules/patch_rules_route.ts | 4 +- .../routes/rules/read_rules_route.ts | 4 +- .../routes/rules/update_rules_route.ts | 4 +- .../rule_execution_log_bootstrapper.ts | 48 -- .../rule_registry_log_client.ts | 42 +- .../rule_types/__mocks__/rule_type.ts | 5 +- .../create_security_rule_type_factory.ts | 5 +- .../create_indicator_match_alert_type.test.ts | 3 - .../create_indicator_match_alert_type.ts | 2 - .../query/create_query_alert_type.test.ts | 2 - .../query/create_query_alert_type.ts | 2 - .../lib/detection_engine/rule_types/types.ts | 8 +- .../security_solution/server/plugin.ts | 99 ++-- .../security_solution/server/routes/index.ts | 4 +- .../server/lib/alerts/test_utils/index.ts | 6 +- x-pack/plugins/uptime/server/plugin.ts | 60 +-- x-pack/plugins/uptime/server/uptime_server.ts | 4 +- .../tests/alerts/rule_registry.ts | 2 +- 88 files changed, 1266 insertions(+), 2596 deletions(-) delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_reader.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/elasticsearch/utils/buffered_stream.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/index.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_query.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/index.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/public_api.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/utils/fields.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/utils/predicates.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.test.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.ts delete mode 100644 x-pack/plugins/rule_registry/server/event_log/utils/utility_types.ts rename x-pack/plugins/rule_registry/server/rule_data_client/{create_rule_data_client_mock.ts => rule_data_client.mock.ts} (61%) create mode 100644 x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts create mode 100644 x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_execution_log_bootstrapper.ts diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts index 022fad6fa7840..db79b4f11df29 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts @@ -8,7 +8,7 @@ import { Observable } from 'rxjs'; import { Logger } from 'kibana/server'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../alerting/server'; -import { RuleDataClient } from '../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../rule_registry/server'; import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; @@ -17,7 +17,7 @@ import { MlPluginSetup } from '../../../../ml/server'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; export interface RegisterRuleDependencies { - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; ml?: MlPluginSetup; alerting: AlertingPluginSetupContract; config$: Observable; diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts index 679f33707b5b5..26667dff90c6e 100644 --- a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -8,7 +8,7 @@ import { Logger } from 'kibana/server'; import { of } from 'rxjs'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; -import type { RuleDataClient } from '../../../../../rule_registry/server'; +import type { IRuleDataClient } from '../../../../../rule_registry/server'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server'; import { APMConfig, APM_SERVER_FEATURE_ID } from '../../..'; @@ -63,7 +63,8 @@ export const createRuleTypeMocks = () => { }; }, isWriteEnabled: jest.fn(() => true), - } as unknown) as RuleDataClient, + indexName: '.alerts-observability.apm.alerts', + } as unknown) as IRuleDataClient, }, services, scheduleActions, diff --git a/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts b/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts index 8981f8b516d97..3b2354a54d96f 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts @@ -8,7 +8,7 @@ import type { EVENT_KIND as EVENT_KIND_TYPED } from '@kbn/rule-data-utils'; // @ts-expect-error import { EVENT_KIND as EVENT_KIND_NON_TYPED } from '@kbn/rule-data-utils/target_node/technical_field_names'; -import { RuleDataClient } from '../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../rule_registry/server'; import { SERVICE_NAME, TRANSACTION_TYPE, @@ -26,7 +26,7 @@ export async function getServiceAlerts({ environment, transactionType, }: { - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; start: number; end: number; serviceName: string; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 24d41e2574bbe..807d21768a50c 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -15,10 +15,10 @@ import { Plugin, PluginInitializerContext, } from 'src/core/server'; -import { isEmpty, mapValues, once } from 'lodash'; +import { isEmpty, mapValues } from 'lodash'; import { SavedObjectsClient } from '../../../../src/core/server'; -import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; +import { Dataset } from '../../rule_registry/server'; import { APMConfig, APMXPackConfig, APM_SERVER_FEATURE_ID } from '.'; import { mergeConfigs } from './index'; import { UI_SETTINGS } from '../../../../src/plugins/data/common'; @@ -106,79 +106,43 @@ export class APMPlugin registerFeaturesUsage({ licensingPlugin: plugins.licensing }); - const { ruleDataService } = plugins.ruleRegistry; const getCoreStart = () => core.getStartServices().then(([coreStart]) => coreStart); - const alertsIndexPattern = ruleDataService.getFullAssetName( - 'observability-apm*' - ); - - const initializeRuleDataTemplates = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName( - 'apm-mappings' - ); - - if (!ruleDataService.isWriteEnabled()) { - return; - } - - await ruleDataService.createOrUpdateComponentTemplate({ - name: componentTemplateName, - body: { - template: { - settings: { - number_of_shards: 1, - }, - mappings: mappingFromFieldMap( - { - [SERVICE_NAME]: { - type: 'keyword', - }, - [SERVICE_ENVIRONMENT]: { - type: 'keyword', - }, - [TRANSACTION_TYPE]: { - type: 'keyword', - }, - [PROCESSOR_EVENT]: { - type: 'keyword', - }, + const { ruleDataService } = plugins.ruleRegistry; + const ruleDataClient = ruleDataService.initializeIndex({ + feature: APM_SERVER_FEATURE_ID, + registrationContext: 'observability.apm', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + version: 0, + mappings: mappingFromFieldMap( + { + [SERVICE_NAME]: { + type: 'keyword', }, - 'strict' - ), - }, - }, - }); - - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName('apm-index-template'), - body: { - index_patterns: [alertsIndexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], + [SERVICE_ENVIRONMENT]: { + type: 'keyword', + }, + [TRANSACTION_TYPE]: { + type: 'keyword', + }, + [PROCESSOR_EVENT]: { + type: 'keyword', + }, + }, + 'strict' + ), }, - }); - await ruleDataService.updateIndexMappingsMatchingPattern( - alertsIndexPattern - ); + ], + indexTemplate: { + version: 0, + }, }); - // initialize eagerly - const initializeRuleDataTemplatesPromise = initializeRuleDataTemplates().catch( - (err) => { - this.logger!.error(err); - } - ); - - const ruleDataClient = ruleDataService.getRuleDataClient( - APM_SERVER_FEATURE_ID, - ruleDataService.getFullAssetName('observability-apm'), - () => initializeRuleDataTemplatesPromise - ); - const resourcePlugins = mapValues(plugins, (value, key) => { return { setup: value, diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index 4279cfd84328c..76f19a6a0ca3e 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -12,7 +12,7 @@ import { KibanaRequest, CoreStart, } from 'src/core/server'; -import { RuleDataClient } from '../../../rule_registry/server'; +import { IRuleDataClient } from '../../../rule_registry/server'; import { AlertingApiRequestHandlerContext } from '../../../alerting/server'; import type { RacApiRequestHandlerContext } from '../../../rule_registry/server'; import { LicensingApiRequestHandlerContext } from '../../../licensing/server'; @@ -72,6 +72,6 @@ export interface APMRouteHandlerResources { start: () => Promise[key]['start']>; }; }; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; telemetryUsageCounter?: TelemetryUsageCounter; } diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts index f67837cff0df1..9b3f7edb97007 100644 --- a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts +++ b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts @@ -5,10 +5,8 @@ * 2.0. */ -import { once } from 'lodash'; import { CoreSetup, Logger } from 'src/core/server'; -import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../../../rule_registry/common/assets'; -import { RuleRegistryPluginSetupContract } from '../../../../rule_registry/server'; +import { Dataset, RuleRegistryPluginSetupContract } from '../../../../rule_registry/server'; import type { InfraFeatureId } from '../../../common/constants'; import { RuleRegistrationContext, RulesServiceStartDeps } from './types'; @@ -25,51 +23,20 @@ export const createRuleDataClient = ({ logger: Logger; ruleDataService: RuleRegistryPluginSetupContract['ruleDataService']; }) => { - const initializeRuleDataTemplates = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName( - `${registrationContext}-mappings` - ); - - const indexNamePattern = ruleDataService.getFullAssetName(`${registrationContext}*`); - - if (!ruleDataService.isWriteEnabled()) { - return; - } - - await ruleDataService.createOrUpdateComponentTemplate({ - name: componentTemplateName, - body: { - template: { - settings: { - number_of_shards: 1, - }, - mappings: {}, - }, - }, - }); - - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName(registrationContext), - body: { - index_patterns: [indexNamePattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], + return ruleDataService.initializeIndex({ + feature: ownerFeatureId, + registrationContext, + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + version: 0, + mappings: {}, }, - }); - - await ruleDataService.updateIndexMappingsMatchingPattern(indexNamePattern); - }); - - // initialize eagerly - const initializeRuleDataTemplatesPromise = initializeRuleDataTemplates().catch((err) => { - logger.error(err); + ], + indexTemplate: { + version: 0, + }, }); - - return ruleDataService.getRuleDataClient( - ownerFeatureId, - ruleDataService.getFullAssetName(registrationContext), - () => initializeRuleDataTemplatesPromise - ); }; diff --git a/x-pack/plugins/infra/server/services/rules/types.ts b/x-pack/plugins/infra/server/services/rules/types.ts index b67b79ee5d3c2..daa44381650e0 100644 --- a/x-pack/plugins/infra/server/services/rules/types.ts +++ b/x-pack/plugins/infra/server/services/rules/types.ts @@ -8,7 +8,7 @@ import { PluginSetupContract as AlertingPluginSetup } from '../../../../alerting/server'; import { createLifecycleExecutor, - RuleDataClient, + IRuleDataClient, RuleRegistryPluginSetupContract, } from '../../../../rule_registry/server'; @@ -24,7 +24,7 @@ export interface RulesServiceStartDeps {} export interface RulesServiceSetup { createLifecycleRuleExecutor: LifecycleRuleExecutorCreator; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; } // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 868e234fcb2a1..f653117113737 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -18,7 +18,7 @@ import { ScopedAnnotationsClientFactory, AnnotationsAPI, } from './lib/annotations/bootstrap_annotations'; -import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server'; +import { Dataset, RuleRegistryPluginSetupContract } from '../../rule_registry/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { uiSettings } from './ui_settings'; import { registerRoutes } from './routes/register_routes'; @@ -100,11 +100,17 @@ export class ObservabilityPlugin implements Plugin { const start = () => core.getStartServices().then(([coreStart]) => coreStart); - const ruleDataClient = plugins.ruleRegistry.ruleDataService.getRuleDataClient( - 'observability', - plugins.ruleRegistry.ruleDataService.getFullAssetName(), - () => Promise.resolve() - ); + const { ruleDataService } = plugins.ruleRegistry; + const ruleDataClient = ruleDataService.initializeIndex({ + feature: 'observability', + registrationContext: 'observability', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [], + indexTemplate: { + version: 0, + }, + }); registerRoutes({ core: { diff --git a/x-pack/plugins/observability/server/routes/register_routes.ts b/x-pack/plugins/observability/server/routes/register_routes.ts index 75b6703cc64de..43b47f26dcdd1 100644 --- a/x-pack/plugins/observability/server/routes/register_routes.ts +++ b/x-pack/plugins/observability/server/routes/register_routes.ts @@ -13,7 +13,7 @@ import { import { CoreSetup, CoreStart, Logger, RouteRegistrar } from 'kibana/server'; import Boom from '@hapi/boom'; import { RequestAbortedError } from '@elastic/elasticsearch/lib/errors'; -import { RuleDataClient } from '../../../rule_registry/server'; +import { IRuleDataClient } from '../../../rule_registry/server'; import { ObservabilityRequestHandlerContext } from '../types'; import { AbstractObservabilityServerRouteRepository } from './types'; @@ -29,7 +29,7 @@ export function registerRoutes({ }; repository: AbstractObservabilityServerRouteRepository; logger: Logger; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; }) { const routes = repository.getRoutes(); diff --git a/x-pack/plugins/observability/server/routes/types.ts b/x-pack/plugins/observability/server/routes/types.ts index 1fa7229c6cf62..15a3274087d0e 100644 --- a/x-pack/plugins/observability/server/routes/types.ts +++ b/x-pack/plugins/observability/server/routes/types.ts @@ -12,7 +12,7 @@ import type { ServerRouteRepository, } from '@kbn/server-route-repository'; import { CoreSetup, CoreStart, KibanaRequest, Logger } from 'kibana/server'; -import { RuleDataClient } from '../../../rule_registry/server'; +import { IRuleDataClient } from '../../../rule_registry/server'; import { ObservabilityServerRouteRepository } from './get_global_observability_server_route_repository'; import { ObservabilityRequestHandlerContext } from '../types'; @@ -24,7 +24,7 @@ export interface ObservabilityRouteHandlerResources { start: () => Promise; setup: CoreSetup; }; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; request: KibanaRequest; context: ObservabilityRequestHandlerContext; logger: Logger; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts index 858b1a5af431f..cc23469524a4e 100644 --- a/x-pack/plugins/rule_registry/common/types.ts +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -7,10 +7,6 @@ import { estypes } from '@elastic/elasticsearch'; -export type PutIndexTemplateRequest = estypes.IndicesPutIndexTemplateRequest & { - body?: { composed_of?: string[] }; -}; - export interface ClusterPutComponentTemplateBody { template: { settings: { diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json index f74bebf585edd..360ea18df9ca1 100644 --- a/x-pack/plugins/rule_registry/kibana.json +++ b/x-pack/plugins/rule_registry/kibana.json @@ -9,7 +9,6 @@ "requiredPlugins": [ "alerting", "data", - "spaces", "triggersActionsUi" ], "optionalPlugins": ["security"], diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts deleted file mode 100644 index 1941208ed07cd..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './index_bootstrapper'; -export * from './index_management_gateway'; -export * from './index_reader'; -export * from './index_writer'; -export * from './resources/ilm_policy'; -export * from './resources/index_mappings'; -export * from './resources/index_names'; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts deleted file mode 100644 index b0c3927cd7dfe..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_bootstrapper.ts +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PublicMethodsOf } from '@kbn/utility-types'; -import { Logger } from 'src/core/server'; - -import { IndexNames } from './resources/index_names'; -import { IndexMappings } from './resources/index_mappings'; -import { createIndexTemplate } from './resources/index_template'; -import { IlmPolicy, defaultIlmPolicy } from './resources/ilm_policy'; -import { IIndexManagementGateway } from './index_management_gateway'; - -interface ConstructorParams { - gateway: IIndexManagementGateway; - logger: Logger; -} - -export interface IndexSpecification { - indexNames: IndexNames; - indexMappings: IndexMappings; - ilmPolicy?: IlmPolicy; -} - -export type IIndexBootstrapper = PublicMethodsOf; - -// TODO: Converge with the logic of .siem-signals index bootstrapping -// x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts - -// TODO: Handle race conditions and potential errors between multiple instances of Kibana -// trying to bootstrap the same index. Possible options: -// - robust idempotent logic with error handling -// - leveraging task_manager to make sure bootstrapping is run only once at a time -// - using some sort of distributed lock -// Maybe we can check how Saved Objects service bootstraps .kibana index - -export class IndexBootstrapper { - private readonly gateway: IIndexManagementGateway; - private readonly logger: Logger; - - constructor(params: ConstructorParams) { - this.gateway = params.gateway; - this.logger = params.logger.get('IndexBootstrapper'); - } - - public async run(indexSpec: IndexSpecification): Promise { - this.logger.debug('bootstrapping elasticsearch resources starting'); - - try { - const { indexNames, indexMappings, ilmPolicy } = indexSpec; - await this.createIlmPolicyIfNotExists(indexNames, ilmPolicy); - await this.createIndexTemplateIfNotExists(indexNames, indexMappings); - await this.createInitialIndexIfNotExists(indexNames); - } catch (err) { - this.logger.error(`error bootstrapping elasticsearch resources: ${err.message}`); - return false; - } - - this.logger.debug('bootstrapping elasticsearch resources complete'); - return true; - } - - private async createIlmPolicyIfNotExists(names: IndexNames, policy?: IlmPolicy): Promise { - const { indexIlmPolicyName } = names; - - const exists = await this.gateway.doesIlmPolicyExist(indexIlmPolicyName); - if (!exists) { - const ilmPolicy = policy ?? defaultIlmPolicy; - await this.gateway.createIlmPolicy(indexIlmPolicyName, ilmPolicy); - } - } - - private async createIndexTemplateIfNotExists( - names: IndexNames, - mappings: IndexMappings - ): Promise { - const { indexTemplateName } = names; - - const templateVersion = 1; // TODO: get from EventSchema definition - const template = createIndexTemplate(names, mappings, templateVersion); - - const exists = await this.gateway.doesIndexTemplateExist(indexTemplateName); - if (!exists) { - await this.gateway.createIndexTemplate(indexTemplateName, template); - } else { - await this.gateway.updateIndexTemplate(indexTemplateName, template); - } - } - - private async createInitialIndexIfNotExists(names: IndexNames): Promise { - const { indexAliasName, indexInitialName } = names; - - const exists = await this.gateway.doesAliasExist(indexAliasName); - if (!exists) { - await this.gateway.createIndex(indexInitialName, { - aliases: { - [indexAliasName]: { - is_write_index: true, - }, - }, - }); - } - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts deleted file mode 100644 index cb04a442d0b34..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_management_gateway.ts +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PublicMethodsOf } from '@kbn/utility-types'; -import { ElasticsearchClient, Logger } from 'src/core/server'; -import { IlmPolicy } from './resources/ilm_policy'; -import { IndexTemplate } from './resources/index_template'; - -interface ConstructorParams { - elasticsearch: Promise; - logger: Logger; -} - -export type IIndexManagementGateway = PublicMethodsOf; - -export class IndexManagementGateway { - private readonly elasticsearch: Promise; - private readonly logger: Logger; - - constructor(params: ConstructorParams) { - this.elasticsearch = params.elasticsearch; - this.logger = params.logger.get('IndexManagementGateway'); - } - - public async doesIlmPolicyExist(policyName: string): Promise { - this.logger.debug(`Checking if ILM policy exists; name="${policyName}"`); - - try { - const es = await this.elasticsearch; - await es.transport.request({ - method: 'GET', - path: `/_ilm/policy/${policyName}`, - }); - } catch (e) { - if (e.statusCode === 404) return false; - throw new Error(`Error checking existence of ILM policy: ${e.message}`); - } - return true; - } - - public async createIlmPolicy(policyName: string, policy: IlmPolicy): Promise { - this.logger.debug(`Creating ILM policy; name="${policyName}"`); - - try { - const es = await this.elasticsearch; - await es.transport.request({ - method: 'PUT', - path: `/_ilm/policy/${policyName}`, - body: policy, - }); - } catch (e) { - throw new Error(`Error creating ILM policy: ${e.message}`); - } - } - - public async doesIndexTemplateExist(templateName: string): Promise { - this.logger.debug(`Checking if index template exists; name="${templateName}"`); - - try { - const es = await this.elasticsearch; - const { body } = await es.indices.existsTemplate({ name: templateName }); - return body as boolean; - } catch (e) { - throw new Error(`Error checking existence of index template: ${e.message}`); - } - } - - public async createIndexTemplate(templateName: string, template: IndexTemplate): Promise { - this.logger.debug(`Creating index template; name="${templateName}"`); - - try { - const es = await this.elasticsearch; - await es.indices.putTemplate({ create: true, name: templateName, body: template }); - } catch (e) { - // The error message doesn't have a type attribute we can look to guarantee it's due - // to the template already existing (only long message) so we'll check ourselves to see - // if the template now exists. This scenario would happen if you startup multiple Kibana - // instances at the same time. - const existsNow = await this.doesIndexTemplateExist(templateName); - if (!existsNow) { - const error = new Error(`Error creating index template: ${e.message}`); - Object.assign(error, { wrapped: e }); - throw error; - } - } - } - - public async updateIndexTemplate(templateName: string, template: IndexTemplate): Promise { - this.logger.debug(`Updating index template; name="${templateName}"`); - - try { - // @ts-expect-error settings is optional - const { settings, ...templateWithoutSettings } = template; - - const es = await this.elasticsearch; - await es.indices.putTemplate({ - create: false, - name: templateName, - body: templateWithoutSettings, - }); - } catch (e) { - throw new Error(`Error updating index template: ${e.message}`); - } - } - - public async doesAliasExist(aliasName: string): Promise { - this.logger.debug(`Checking if index alias exists; name="${aliasName}"`); - - try { - const es = await this.elasticsearch; - const { body } = await es.indices.existsAlias({ name: aliasName }); - return body as boolean; - } catch (e) { - throw new Error(`Error checking existence of initial index: ${e.message}`); - } - } - - public async createIndex(indexName: string, body: Record = {}): Promise { - this.logger.debug(`Creating index; name="${indexName}"`); - this.logger.debug(JSON.stringify(body, null, 2)); - - try { - const es = await this.elasticsearch; - await es.indices.create({ - index: indexName, - body, - }); - } catch (e) { - if (e.body?.error?.type !== 'resource_already_exists_exception') { - this.logger.error(e); - this.logger.error(JSON.stringify(e.meta, null, 2)); - throw new Error(`Error creating initial index: ${e.message}`); - } - } - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_reader.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_reader.ts deleted file mode 100644 index 84c0b41f7e1a0..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_reader.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PublicMethodsOf } from '@kbn/utility-types'; -import { estypes } from '@elastic/elasticsearch'; -import { Logger, ElasticsearchClient } from 'src/core/server'; - -interface ConstructorParams { - indexName: string; - elasticsearch: Promise; - logger: Logger; -} - -export type IIndexReader = PublicMethodsOf; - -export class IndexReader { - private readonly indexName: string; - private readonly elasticsearch: Promise; - private readonly logger: Logger; - - constructor(params: ConstructorParams) { - this.indexName = params.indexName; - this.elasticsearch = params.elasticsearch; - this.logger = params.logger.get('IndexReader'); - } - - public async search(request: estypes.SearchRequest) { - const requestToSend: estypes.SearchRequest = { - ...request, - index: this.indexName, - }; - - this.logger.debug(`Searching; request: ${JSON.stringify(requestToSend, null)}`); - - const esClient = await this.elasticsearch; - const response = await esClient.search(requestToSend); - - return response; - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts deleted file mode 100644 index 6fd1c954d8c14..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index_writer.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PublicMethodsOf } from '@kbn/utility-types'; -import util from 'util'; -import { Logger, ElasticsearchClient } from 'src/core/server'; -import { BufferedStream } from './utils/buffered_stream'; - -type Document = Record; - -interface BufferItem { - index: string; - doc: Document; -} - -interface ConstructorParams { - indexName: string; - elasticsearch: Promise; - isWriteEnabled: boolean; - logger: Logger; -} - -export type IIndexWriter = PublicMethodsOf; - -export class IndexWriter { - private readonly indexName: string; - private readonly elasticsearch: Promise; - private readonly isWriteEnabled: boolean; - private readonly logger: Logger; - private readonly buffer: BufferedStream; - - constructor(params: ConstructorParams) { - this.indexName = params.indexName; - this.elasticsearch = params.elasticsearch; - this.isWriteEnabled = params.isWriteEnabled; - this.logger = params.logger.get('IndexWriter'); - - this.buffer = new BufferedStream({ - flush: (items) => this.bulkIndex(items), - }); - } - - public indexOne(doc: Document): void { - if (this.isWriteEnabled) { - this.logger.debug('Buffering 1 document'); - this.buffer.enqueue({ index: this.indexName, doc }); - } - } - - public indexMany(docs: Document[]): void { - if (this.isWriteEnabled) { - this.logger.debug(`Buffering ${docs.length} documents`); - docs.forEach((doc) => { - this.buffer.enqueue({ index: this.indexName, doc }); - }); - } - } - - public async close(): Promise { - await this.buffer.closeAndWaitUntilFlushed(); - } - - private async bulkIndex(items: BufferItem[]): Promise { - this.logger.debug(`Indexing ${items.length} documents`); - - const bulkBody: Array> = []; - - for (const item of items) { - if (item.doc === undefined) continue; - - bulkBody.push({ create: { _index: item.index, version: 1 } }); - bulkBody.push(item.doc); - } - - try { - const es = await this.elasticsearch; - const response = await es.bulk({ body: bulkBody }); - - if (response.body.errors) { - const error = new Error('Error writing some bulk events'); - error.stack += '\n' + util.inspect(response.body.items, { depth: null }); - this.logger.error(error); - } - } catch (e) { - this.logger.error( - `error writing bulk events: "${e.message}"; docs: ${JSON.stringify(bulkBody)}` - ); - } - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts deleted file mode 100644 index 7663ea27fb0df..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/ilm_policy.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; - -export interface IlmPolicy { - policy: estypes.IlmPolicy; -} - -export const defaultIlmPolicy: IlmPolicy = { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_age: '90d', - max_size: '50gb', - }, - }, - }, - delete: { - actions: { - delete: {}, - }, - }, - }, - }, -}; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts deleted file mode 100644 index 064bde5001f7b..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_mappings.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface IndexMappings { - dynamic: 'strict' | boolean; - properties: Record; - _meta?: Record; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts deleted file mode 100644 index 1082c62b95e70..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_names.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface IndexParams { - /** @example '.alerts' */ - indexPrefix: string; - - /** @example 'security', 'security.alerts', 'observability.events' */ - logName: string; - - /** @example 'default' */ - kibanaSpaceId: string; -} - -export interface IndexNames extends IndexParams { - /** @example '.alerts-security.alerts' */ - indexBaseName: string; - - /** @example '.alerts-security.alerts-*' */ - indexBasePattern: string; - - /** @example '.alerts-security.alerts-default' */ - indexAliasName: string; - - /** @example '.alerts-security.alerts-default-*' */ - indexAliasPattern: string; - - /** @example '.alerts-security.alerts-default-policy' */ - indexIlmPolicyName: string; - - /** @example '.alerts-security.alerts-default-template' */ - indexTemplateName: string; - - /** @example '.alerts-security.alerts-default-000001' */ - indexInitialName: string; -} - -export abstract class IndexNames { - public static create(params: IndexParams): IndexNames { - const { indexPrefix, logName, kibanaSpaceId } = params; - - // TODO: validate params - - const indexBaseName = joinWithDash(indexPrefix, logName); - const indexBasePattern = joinWithDash(indexPrefix, logName, '*'); - const indexAliasName = joinWithDash(indexPrefix, logName, kibanaSpaceId); - const indexAliasPattern = joinWithDash(indexPrefix, logName, kibanaSpaceId, '*'); - const indexIlmPolicyName = joinWithDash(indexPrefix, logName, kibanaSpaceId, 'policy'); - const indexTemplateName = joinWithDash(indexPrefix, logName, kibanaSpaceId, 'template'); - const indexInitialName = joinWithDash(indexPrefix, logName, kibanaSpaceId, '000001'); - - return { - indexPrefix, - logName, - kibanaSpaceId, - indexBaseName, - indexBasePattern, - indexAliasName, - indexAliasPattern, - indexIlmPolicyName, - indexTemplateName, - indexInitialName, - }; - } - - public static createChild(parent: IndexNames, logName: string): IndexNames { - return this.create({ - indexPrefix: parent.indexPrefix, - logName: this.createChildLogName(parent.logName, logName), - kibanaSpaceId: parent.kibanaSpaceId, - }); - } - - public static createChildLogName(parentLogName: string, logName: string): string { - return joinWithDot(parentLogName, logName); - } -} - -const joinWithDash = (...names: string[]): string => names.join('-'); -const joinWithDot = (...names: string[]): string => names.join('.'); diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts deleted file mode 100644 index 0045bce3ed873..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/resources/index_template.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { IndexNames } from './index_names'; -import { IndexMappings } from './index_mappings'; - -export type IndexTemplate = estypes.IndicesPutTemplateRequest['body']; - -export const createIndexTemplate = ( - names: IndexNames, - mappings: IndexMappings, - version: number -): IndexTemplate => { - const { indexAliasName, indexAliasPattern, indexIlmPolicyName } = names; - - return { - index_patterns: [indexAliasPattern], - settings: { - number_of_shards: 1, // TODO: do we need to set this? - auto_expand_replicas: '0-1', // TODO: do we need to set? - index: { - lifecycle: { - name: indexIlmPolicyName, - rollover_alias: indexAliasName, - }, - }, - mapping: { - total_fields: { - limit: 10000, - }, - }, - sort: { - field: '@timestamp', - order: 'desc', - }, - }, - // @ts-expect-error IndexMappings is not assignale to estypes.MappingTypeMapping - mappings: { - ...mappings, - _meta: { - ...mappings._meta, - version, - }, - }, - version, - }; -}; diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/utils/buffered_stream.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/utils/buffered_stream.ts deleted file mode 100644 index d968cd5a0ac68..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/utils/buffered_stream.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Subject } from 'rxjs'; -import { bufferTime, filter as rxFilter, switchMap } from 'rxjs/operators'; - -export const DEFAULT_BUFFER_TIME_MS = 1000; -export const DEFAULT_BUFFER_SIZE = 100; - -interface ConstructorParams { - maxBufferTimeMs?: number; - maxBufferSize?: number; - flush: (items: TItem[]) => Promise; -} - -// TODO: handle possible exceptions in flush and maybe add retry logic - -export class BufferedStream { - private readonly buffer$: Subject; - private readonly whenBufferCompleteAndFlushed: Promise; - - constructor(params: ConstructorParams) { - const maxBufferTime = params.maxBufferTimeMs ?? DEFAULT_BUFFER_TIME_MS; - const maxBufferSize = params.maxBufferSize ?? DEFAULT_BUFFER_SIZE; - - this.buffer$ = new Subject(); - - // Buffer items for time/buffer length, ignore empty buffers, - // then flush the buffered items; kick things off with a promise - // on the observable, which we'll wait on in shutdown - this.whenBufferCompleteAndFlushed = this.buffer$ - .pipe( - bufferTime(maxBufferTime, null, maxBufferSize), - rxFilter((docs) => docs.length > 0), - switchMap(async (docs) => await params.flush(docs)) - ) - .toPromise(); - } - - public enqueue(item: TItem): void { - this.buffer$.next(item); - } - - public async closeAndWaitUntilFlushed(): Promise { - this.buffer$.complete(); - await this.whenBufferCompleteAndFlushed; - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts b/x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts deleted file mode 100644 index 77c041a4059b5..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './schema_types'; -export * from './schema'; diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts b/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts deleted file mode 100644 index 6ab82e7027efc..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EventSchema, Event } from './schema_types'; -import { FieldMap, runtimeTypeFromFieldMap, mergeFieldMaps } from '../../../common/field_map'; -import { - TechnicalRuleFieldMap, - technicalRuleFieldMap, -} from '../../../common/assets/field_maps/technical_rule_field_map'; - -const baseSchema = createSchema(technicalRuleFieldMap); - -export abstract class Schema { - public static create(fields: TMap): EventSchema { - return createSchema(fields); - } - - public static combine( - s1: EventSchema, - s2: EventSchema - ): EventSchema { - const combinedFields = mergeFieldMaps(s1.objectFields, s2.objectFields); - return createSchema(combinedFields); - } - - public static getBase(): EventSchema { - return baseSchema; - } - - public static extendBase( - fields: TMap - ): EventSchema { - const extensionSchema = createSchema(fields); - return this.combine(baseSchema, extensionSchema); - } -} - -function createSchema(fields: TMap): EventSchema { - const objectType: Event = ({} as unknown) as Event; - const runtimeType = runtimeTypeFromFieldMap(fields); - - return { - objectFields: fields, - objectType, - runtimeType, - }; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts b/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts deleted file mode 100644 index e5c665652fe97..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/event_schema/schema_types.ts +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FieldMap, FieldMapType, TypeOfFieldMap } from '../../../common/field_map'; - -export interface EventSchema { - objectFields: TMap; - objectType: Event; - runtimeType: EventRuntimeType; -} - -export type Event = TypeOfFieldMap; - -export type EventRuntimeType = FieldMapType; - -export { FieldMap }; diff --git a/x-pack/plugins/rule_registry/server/event_log/index.ts b/x-pack/plugins/rule_registry/server/event_log/index.ts deleted file mode 100644 index cf7467588c22f..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './elasticsearch'; -export * from './event_schema'; -export * from './log'; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts deleted file mode 100644 index 2b1ecde48d2db..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { DeepPartial } from '../utils/utility_types'; -import { IndexNames } from '../elasticsearch'; -import { IEventLog, IEventLogger, IEventLoggerTemplate, IEventQueryBuilder } from './public_api'; -import { EventLogParams } from './internal_api'; -import { EventLoggerTemplate } from './event_logger_template'; -import { EventQueryBuilder } from './event_query_builder'; - -export class EventLog implements IEventLog { - private readonly params: EventLogParams; - private readonly initialTemplate: IEventLoggerTemplate; - - constructor(params: EventLogParams) { - this.params = params; - this.initialTemplate = new EventLoggerTemplate({ - ...params, - eventLoggerName: '', - eventFields: {}, - }); - } - - public getNames(): IndexNames { - return this.params.indexNames; - } - - public getLoggerTemplate(fields: DeepPartial): IEventLoggerTemplate { - return this.initialTemplate.getLoggerTemplate(fields); - } - - public getLogger(loggerName: string, fields?: DeepPartial): IEventLogger { - return this.initialTemplate.getLogger(loggerName, fields); - } - - public getQueryBuilder(): IEventQueryBuilder { - return new EventQueryBuilder(this.params); - } - - public async search( - request: estypes.SearchRequest - ): Promise> { - const response = await this.params.indexReader.search(request); - return response.body; - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts deleted file mode 100644 index 0498a7cd97b2f..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_bootstrapper.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from 'kibana/server'; -import { IIndexBootstrapper, IndexSpecification } from '../elasticsearch'; - -interface ConstructorParams { - indexSpec: IndexSpecification; - indexBootstrapper: IIndexBootstrapper; - isWriteEnabled: boolean; - logger: Logger; -} - -export class EventLogBootstrapper { - private readonly indexSpec: IndexSpecification; - private readonly indexBootstrapper: IIndexBootstrapper; - private readonly logger: Logger; - private readonly isWriteEnabled: boolean; - private isIndexBootstrapped: boolean; - - constructor(params: ConstructorParams) { - this.indexSpec = params.indexSpec; - this.indexBootstrapper = params.indexBootstrapper; - this.logger = params.logger.get('EventLogBootstrapper'); - this.isWriteEnabled = params.isWriteEnabled; - this.isIndexBootstrapped = false; - } - - public async run(): Promise { - if (this.isIndexBootstrapped || !this.isWriteEnabled) { - return; - } - - const { logName, indexAliasName } = this.indexSpec.indexNames; - const logInfo = `log="${logName}" index="${indexAliasName}"`; - - this.logger.debug(`Bootstrapping started, ${logInfo}`); - this.isIndexBootstrapped = await this.indexBootstrapper.run(this.indexSpec); - this.logger.debug( - `Bootstrapping ${this.isIndexBootstrapped ? 'succeeded' : 'failed'}, ${logInfo}` - ); - - if (!this.isIndexBootstrapped) { - throw new Error(`Event log bootstrapping failed, ${logInfo}`); - } - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts deleted file mode 100644 index 124664d5578b0..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_definition.ts +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IlmPolicy, defaultIlmPolicy, IndexNames } from '../elasticsearch'; -import { EventSchema, FieldMap, Schema } from '../event_schema'; -import { EventLogOptions, IEventLogDefinition } from './public_api'; - -export class EventLogDefinition implements IEventLogDefinition { - public readonly eventLogName: string; - public readonly eventSchema: EventSchema; - public readonly ilmPolicy: IlmPolicy; - - constructor(options: EventLogOptions) { - // TODO: validate options; options.name should not contain "-" and "." - this.eventLogName = options.name; - this.eventSchema = options.schema; - this.ilmPolicy = options.ilmPolicy ?? defaultIlmPolicy; - } - - public defineChild( - options: EventLogOptions - ): IEventLogDefinition { - const childName = IndexNames.createChildLogName(this.eventLogName, options.name); - const childSchema = Schema.combine(this.eventSchema, options.schema); - const childPolicy = options.ilmPolicy ?? this.ilmPolicy; - - return new EventLogDefinition({ - name: childName, - schema: childSchema, - ilmPolicy: childPolicy, - }); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts deleted file mode 100644 index d1ecd6a977a08..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_provider.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IIndexWriter } from '../elasticsearch'; -import { IEventLog } from './public_api'; -import { IEventLogProvider } from './internal_api'; -import { EventLogBootstrapper } from './event_log_bootstrapper'; - -interface ConstructorParams { - log: IEventLog; - logBootstrapper: EventLogBootstrapper; - indexWriter: IIndexWriter; -} - -export class EventLogProvider implements IEventLogProvider { - constructor(private readonly params: ConstructorParams) {} - - public getLog(): IEventLog { - return this.params.log; - } - - public async bootstrapLog(): Promise { - await this.params.logBootstrapper.run(); - } - - public async shutdownLog(): Promise { - await this.params.indexWriter.close(); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts deleted file mode 100644 index 52f6c6bd918d4..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_registry.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Event, FieldMap } from '../event_schema'; -import { IEventLogDefinition } from './public_api'; -import { IEventLogRegistry, IEventLogProvider } from './internal_api'; - -const getRegistryKey = (definition: IEventLogDefinition, spaceId: string) => - `${definition.eventLogName}-${spaceId}`; - -interface RegistryEntry { - definition: IEventLogDefinition; - spaceId: string; - provider: IEventLogProvider; -} - -export class EventLogRegistry implements IEventLogRegistry { - private readonly map = new Map(); - - public get( - definition: IEventLogDefinition, - spaceId: string - ): IEventLogProvider> | null { - const key = getRegistryKey(definition, spaceId); - const entry = this.map.get(key); - return entry != null ? (entry.provider as IEventLogProvider>) : null; - } - - public add( - definition: IEventLogDefinition, - spaceId: string, - provider: IEventLogProvider> - ): void { - const key = getRegistryKey(definition, spaceId); - - if (this.map.has(key)) { - throw new Error(`Event log already registered, key="${key}"`); - } - - this.map.set(key, { - definition, - spaceId, - provider, - }); - } - - public async shutdown(): Promise { - const entries = Array.from(this.map.values()); - const promises = entries.map(({ provider }) => provider.shutdownLog()); - await Promise.all(promises); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts deleted file mode 100644 index 6f1d4aba9f6e5..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_resolver.ts +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - IndexBootstrapper, - IndexManagementGateway, - IndexNames, - IndexReader, - IndexSpecification, - IndexWriter, -} from '../elasticsearch'; - -import { Event, FieldMap } from '../event_schema'; -import { IEventLogRegistry, IEventLogProvider } from './internal_api'; -import { - EventLogServiceConfig, - EventLogServiceDependencies, - IEventLog, - IEventLogDefinition, - IEventLogResolver, -} from './public_api'; - -import { EventLog } from './event_log'; -import { EventLogBootstrapper } from './event_log_bootstrapper'; -import { EventLogProvider } from './event_log_provider'; -import { mappingFromFieldMap } from './utils/mapping_from_field_map'; - -export class EventLogResolver implements IEventLogResolver { - private readonly indexBootstrapper: IndexBootstrapper; - - constructor( - private readonly config: EventLogServiceConfig, - private readonly deps: EventLogServiceDependencies, - private readonly registry: IEventLogRegistry, - private readonly bootstrapLog: boolean - ) { - this.indexBootstrapper = this.createIndexBootstrapper(); - } - - public async resolve( - definition: IEventLogDefinition, - kibanaSpaceId: string - ): Promise>> { - const provider = this.resolveLogProvider(definition, kibanaSpaceId); - - if (this.bootstrapLog) { - await provider.bootstrapLog(); - } - - return provider.getLog(); - } - - private resolveLogProvider( - definition: IEventLogDefinition, - kibanaSpaceId: string - ): IEventLogProvider> { - const existingProvider = this.registry.get(definition, kibanaSpaceId); - if (existingProvider) { - return existingProvider; - } - - const indexSpec = this.createIndexSpec(definition, kibanaSpaceId); - const indexReader = this.createIndexReader(indexSpec); - const indexWriter = this.createIndexWriter(indexSpec); - const logBootstrapper = this.createEventLogBootstrapper(indexSpec); - const log = this.createEventLog(indexSpec, indexReader, indexWriter); - const logProvider = new EventLogProvider({ - log, - logBootstrapper, - indexWriter, - }); - - this.registry.add(definition, kibanaSpaceId, logProvider); - - return logProvider; - } - - private createIndexSpec( - definition: IEventLogDefinition, - kibanaSpaceId: string - ): IndexSpecification { - const { indexPrefix } = this.config; - const { eventLogName, eventSchema, ilmPolicy } = definition; - - const indexNames = IndexNames.create({ - indexPrefix, - logName: eventLogName, - kibanaSpaceId, - }); - - const indexMappings = mappingFromFieldMap(eventSchema.objectFields, 'strict'); - - return { indexNames, indexMappings, ilmPolicy }; - } - - private createIndexBootstrapper(): IndexBootstrapper { - const { clusterClient, logger } = this.deps; - - return new IndexBootstrapper({ - gateway: new IndexManagementGateway({ - elasticsearch: clusterClient.then((c) => c.asInternalUser), - logger, - }), - logger, - }); - } - - private createIndexReader(indexSpec: IndexSpecification): IndexReader { - const { clusterClient, logger } = this.deps; - const { indexNames } = indexSpec; - - return new IndexReader({ - indexName: indexNames.indexAliasPattern, - elasticsearch: clusterClient.then((c) => c.asInternalUser), // TODO: internal or current? - logger, - }); - } - - private createIndexWriter(indexSpec: IndexSpecification): IndexWriter { - const { clusterClient, logger } = this.deps; - const { isWriteEnabled } = this.config; - const { indexNames } = indexSpec; - - return new IndexWriter({ - indexName: indexNames.indexAliasName, - elasticsearch: clusterClient.then((c) => c.asInternalUser), // TODO: internal or current? - isWriteEnabled, - logger, - }); - } - - private createEventLogBootstrapper(indexSpec: IndexSpecification): EventLogBootstrapper { - const { logger } = this.deps; - const { isWriteEnabled } = this.config; - - return new EventLogBootstrapper({ - indexSpec, - indexBootstrapper: this.indexBootstrapper, - isWriteEnabled, - logger, - }); - } - - private createEventLog( - indexSpec: IndexSpecification, - indexReader: IndexReader, - indexWriter: IndexWriter - ): IEventLog> { - const { logger } = this.deps; - - return new EventLog>({ - indexNames: indexSpec.indexNames, - indexReader, - indexWriter, - logger, - }); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts deleted file mode 100644 index b5b1d23f2e215..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_log_service.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { KibanaRequest } from 'kibana/server'; - -import { Event, FieldMap } from '../event_schema'; -import { - EventLogServiceConfig, - EventLogServiceDependencies, - IEventLog, - IEventLogDefinition, - IEventLogResolver, - IEventLogService, - IScopedEventLogResolver, -} from './public_api'; - -import { EventLogRegistry } from './event_log_registry'; -import { EventLogResolver } from './event_log_resolver'; - -const BOOTSTRAP_BY_DEFAULT = true; - -interface ConstructorParams { - config: EventLogServiceConfig; - dependencies: EventLogServiceDependencies; -} - -export class EventLogService implements IEventLogService { - private readonly registry: EventLogRegistry; - - constructor(private readonly params: ConstructorParams) { - this.registry = new EventLogRegistry(); - } - - public getResolver(bootstrapLog = BOOTSTRAP_BY_DEFAULT): IEventLogResolver { - const { params, registry } = this; - const { config, dependencies } = params; - - return new EventLogResolver(config, dependencies, registry, bootstrapLog); - } - - public getScopedResolver( - request: KibanaRequest, - bootstrapLog = BOOTSTRAP_BY_DEFAULT - ): IScopedEventLogResolver { - const resolver = this.getResolver(bootstrapLog); - - return { - resolve: async ( - definition: IEventLogDefinition - ): Promise>> => { - const spaces = await this.params.dependencies.spacesService; - const spaceId = spaces.getSpaceId(request); - - const log = await resolver.resolve(definition, spaceId); - return log; - }, - }; - } - - public async stop(): Promise { - await this.registry.shutdown(); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts deleted file mode 100644 index c6f88f49835d7..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_logger.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DeepPartial } from '../utils/utility_types'; -import { mergeFields } from '../utils/fields'; -import { EventLoggerParams } from './internal_api'; -import { IEventLogger, IEventLoggerTemplate } from './public_api'; - -export class EventLogger implements IEventLogger { - private readonly params: EventLoggerParams; - private readonly ownTemplate: IEventLoggerTemplate; - - constructor(params: EventLoggerParams, template: IEventLoggerTemplate) { - this.params = params; - this.ownTemplate = template; - } - - public getLoggerTemplate(fields: DeepPartial): IEventLoggerTemplate { - return this.ownTemplate.getLoggerTemplate(fields); - } - - public getLogger(name: string, fields?: DeepPartial): IEventLogger { - return this.ownTemplate.getLogger(name, fields); - } - - public logEvent(fields: DeepPartial): void { - const { eventFields, indexWriter } = this.params; - - const event = mergeFields(eventFields, fields); - indexWriter.indexOne(event); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts deleted file mode 100644 index 57d2dabb47bfa..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_logger_template.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { DeepPartial } from '../utils/utility_types'; -import { mergeFields } from '../utils/fields'; -import { IEventLogger, IEventLoggerTemplate } from './public_api'; -import { EventLoggerParams } from './internal_api'; -import { EventLogger } from './event_logger'; - -export class EventLoggerTemplate implements IEventLoggerTemplate { - private readonly params: EventLoggerParams; - - constructor(params: EventLoggerParams) { - this.params = params; - } - - public getLoggerTemplate(fields: DeepPartial): IEventLoggerTemplate { - const nextParams = this.getNextParams('', fields); - return new EventLoggerTemplate(nextParams); - } - - public getLogger(name: string, fields?: DeepPartial): IEventLogger { - const nextParams = this.getNextParams(name, fields); - const nextTemplate = new EventLoggerTemplate(nextParams); - return new EventLogger(nextParams, nextTemplate); - } - - private getNextParams( - extName: string, - extFields?: DeepPartial - ): EventLoggerParams { - const { indexNames, eventLoggerName, eventFields } = this.params; - - const baseName = eventLoggerName; - const nextName = [baseName, extName].filter(Boolean).join('.'); - - const baseFields = eventFields; - const nextFields = mergeFields(baseFields, extFields, { - // TODO: Define a schema for own fields used/set by event log. Add it to the base schema. - // Then maybe introduce a base type for TEvent. - 'kibana.event_log.log_name': indexNames.logName, - 'kibana.event_log.logger_name': nextName, - } as any); - - return { - ...this.params, - eventLoggerName: nextName, - eventFields: nextFields, - }; - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts deleted file mode 100644 index 0eabe4be64837..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_query.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { IIndexReader } from '../elasticsearch'; -import { truthy } from '../utils/predicates'; -import { IEventQuery } from './public_api'; - -export interface EventQueryParams { - indexReader: IIndexReader; - request: estypes.SearchRequest; -} - -export class EventQuery implements IEventQuery { - constructor(private readonly params: EventQueryParams) {} - - public async execute(): Promise { - const { indexReader, request } = this.params; - - const response = await indexReader.search(request); - return response.body.hits.hits.map((hit) => hit._source).filter(truthy); - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts b/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts deleted file mode 100644 index 9fafaa8ed1b76..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/event_query_builder.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { getFlattenedObject } from '@kbn/std'; -import { estypes } from '@elastic/elasticsearch'; -import { esKuery } from '../../../../../../src/plugins/data/server'; - -import { DeepPartial } from '../utils/utility_types'; -import { mergeFields } from '../utils/fields'; -import { EventLogParams } from './internal_api'; -import { IEventQueryBuilder, IEventQuery, SortingParams, PaginationParams } from './public_api'; -import { EventQuery } from './event_query'; - -export class EventQueryBuilder implements IEventQueryBuilder { - private readonly params: EventLogParams; - private loggerName: string; - private fields: DeepPartial | null; - private kql: string; - private sorting: SortingParams; - private pagination: PaginationParams; - - constructor(params: EventLogParams) { - this.params = params; - this.loggerName = ''; - this.fields = null; - this.kql = ''; - this.sorting = [{ '@timestamp': { order: 'desc' } }, { 'event.sequence': { order: 'desc' } }]; - this.pagination = { page: 1, perPage: 20 }; - } - - public filterByLogger(loggerName: string): IEventQueryBuilder { - this.loggerName = loggerName; - return this; - } - - public filterByFields(fields: DeepPartial): IEventQueryBuilder { - this.fields = mergeFields(this.fields ?? {}, fields); - return this; - } - - public filterByKql(kql: string): IEventQueryBuilder { - this.kql = kql; - return this; - } - - public sortBy(params: SortingParams): IEventQueryBuilder { - this.sorting = params; - return this; - } - - public paginate(params: PaginationParams): IEventQueryBuilder { - this.pagination = params; - return this; - } - - public buildQuery(): IEventQuery { - const { indexReader } = this.params; - const { page, perPage } = this.pagination; - - const request: estypes.SearchRequest = { - track_total_hits: true, - body: { - from: (page - 1) * perPage, - size: perPage, - sort: this.sorting, - query: { - bool: { - filter: this.buildFilter(), - }, - }, - }, - }; - - return new EventQuery({ indexReader, request }); - } - - private buildFilter(): estypes.QueryDslQueryContainer[] { - const result: estypes.QueryDslQueryContainer[] = []; - - if (this.loggerName) { - result.push({ - term: { 'kibana.event_log.logger_name': this.loggerName }, - }); - } - - if (this.fields) { - const flatFields = getFlattenedObject(this.fields); - Object.entries(flatFields) - .map(([key, value]) => { - const queryName = Array.isArray(value) ? 'terms' : 'term'; - return { [queryName]: { [key]: value } }; - }) - .forEach((query) => { - result.push(query); - }); - } - - if (this.kql) { - const dsl = esKuery.toElasticsearchQuery(esKuery.fromKueryExpression(this.kql)); - const queries = Array.isArray(dsl) ? dsl : [dsl]; - result.push(...queries); - } - - return result; - } -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/index.ts b/x-pack/plugins/rule_registry/server/event_log/log/index.ts deleted file mode 100644 index e5593390733e4..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export * from './event_log_definition'; -export * from './event_log_service'; -export * from './public_api'; diff --git a/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts b/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts deleted file mode 100644 index 8db931b35912d..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/internal_api.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { Logger } from 'kibana/server'; - -import { IIndexReader, IIndexWriter, IndexNames } from '../elasticsearch'; -import { Event, FieldMap } from '../event_schema'; -import { DeepPartial } from '../utils/utility_types'; -import { IEventLogDefinition, IEventLog } from './public_api'; - -export interface IEventLogRegistry { - get( - definition: IEventLogDefinition, - spaceId: string - ): IEventLogProvider> | null; - - add( - definition: IEventLogDefinition, - spaceId: string, - provider: IEventLogProvider> - ): void; - - shutdown(): Promise; -} - -export interface IEventLogProvider { - getLog(): IEventLog; - bootstrapLog(): Promise; - shutdownLog(): Promise; -} - -export interface EventLogParams { - indexNames: IndexNames; - indexReader: IIndexReader; - indexWriter: IIndexWriter; - logger: Logger; -} - -export interface EventLoggerParams extends EventLogParams { - eventLoggerName: string; - eventFields: DeepPartial; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts b/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts deleted file mode 100644 index 7dcaee9d382b1..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/public_api.ts +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { estypes } from '@elastic/elasticsearch'; -import { IClusterClient, KibanaRequest, Logger } from 'kibana/server'; -import { SpacesServiceStart } from '../../../../spaces/server'; - -import { IlmPolicy, IndexNames, IndexSpecification } from '../elasticsearch'; -import { FieldMap, Event, EventSchema } from '../event_schema'; -import { DeepPartial } from '../utils/utility_types'; - -export { IlmPolicy, IndexSpecification }; - -// ------------------------------------------------------------------------------------------------- -// Definition API (defining log hierarchies as simple objects) - -export interface EventLogOptions { - name: string; - schema: EventSchema; - ilmPolicy?: IlmPolicy; -} - -export interface IEventLogDefinition { - eventLogName: string; - eventSchema: EventSchema; - ilmPolicy: IlmPolicy; - - defineChild( - options: EventLogOptions - ): IEventLogDefinition; -} - -// ------------------------------------------------------------------------------------------------- -// Resolving and bootstrapping API (creating runtime objects representing logs, bootstrapping indices) - -export interface EventLogServiceConfig { - indexPrefix: string; - isWriteEnabled: boolean; -} - -export interface EventLogServiceDependencies { - clusterClient: Promise; - spacesService: Promise; - logger: Logger; -} - -export interface IEventLogService { - getResolver(bootstrapLog?: boolean): IEventLogResolver; - getScopedResolver(request: KibanaRequest, bootstrapLog?: boolean): IScopedEventLogResolver; -} - -export interface IEventLogResolver { - resolve( - definition: IEventLogDefinition, - spaceId: string - ): Promise>>; -} - -export interface IScopedEventLogResolver { - resolve( - definition: IEventLogDefinition - ): Promise>>; -} - -export interface IEventLog extends IEventLoggerTemplate { - getNames(): IndexNames; - - getQueryBuilder(): IEventQueryBuilder; - - search( - request: estypes.SearchRequest - ): Promise>; -} - -// ------------------------------------------------------------------------------------------------- -// Write API (logging events) - -export interface IEventLoggerTemplate { - getLoggerTemplate(fields: DeepPartial): IEventLoggerTemplate; - getLogger(name: string, fields?: DeepPartial): IEventLogger; -} - -export interface IEventLogger extends IEventLoggerTemplate { - logEvent(fields: DeepPartial): void; -} - -// ------------------------------------------------------------------------------------------------- -// Read API (searching, filtering, sorting, pagination, aggregation over events) - -export interface IEventQueryBuilder { - filterByLogger(loggerName: string): IEventQueryBuilder; - filterByFields(fields: DeepPartial): IEventQueryBuilder; - filterByKql(kql: string): IEventQueryBuilder; - sortBy(params: SortingParams): IEventQueryBuilder; - paginate(params: PaginationParams): IEventQueryBuilder; - - buildQuery(): IEventQuery; -} - -export type SortingParams = estypes.SearchSort; - -export interface PaginationParams { - page: number; - perPage: number; -} - -export interface IEventQuery { - execute(): Promise; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts deleted file mode 100644 index 02c759cc328d2..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/log/utils/mapping_from_field_map.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { set } from '@elastic/safer-lodash-set'; -import { FieldMap } from '../../../../common/field_map'; -import { IndexMappings } from '../../elasticsearch'; - -export function mappingFromFieldMap( - fieldMap: FieldMap, - dynamic: 'strict' | boolean -): IndexMappings { - const mappings = { - dynamic, - properties: {}, - }; - - const fields = Object.keys(fieldMap).map((key) => { - const field = fieldMap[key]; - return { - name: key, - ...field, - }; - }); - - fields.forEach((field) => { - const { name, required, array, ...rest } = field; - - set(mappings.properties, field.name.split('.').join('.properties.'), rest); - }); - - return mappings; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/utils/fields.ts b/x-pack/plugins/rule_registry/server/event_log/utils/fields.ts deleted file mode 100644 index 4f140cfb3434f..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/utils/fields.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { merge } from 'lodash'; -import { DeepPartial } from './utility_types'; - -export const mergeFields = ( - base: DeepPartial, - ext1?: DeepPartial, - ext2?: DeepPartial, - ext3?: DeepPartial -): DeepPartial => { - return merge({}, base, ext1 ?? {}, ext2 ?? {}, ext3 ?? {}); -}; diff --git a/x-pack/plugins/rule_registry/server/event_log/utils/predicates.ts b/x-pack/plugins/rule_registry/server/event_log/utils/predicates.ts deleted file mode 100644 index 40daac1fedcc6..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/utils/predicates.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export function nonNullable(value: T): value is NonNullable { - return value !== null && value !== undefined; -} - -export type Truthy = T extends false | '' | 0 | null | undefined ? never : T; // from lodash - -export function truthy(value: T): value is Truthy { - return Boolean(value); -} diff --git a/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.test.ts b/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.test.ts deleted file mode 100644 index f01d438ce79a0..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { createReadySignal, ReadySignal } from './ready_signal'; - -describe('ReadySignal', () => { - let readySignal: ReadySignal; - - beforeEach(() => { - readySignal = createReadySignal(); - }); - - test('works as expected', async () => { - readySignal.signal(42); - const ready = await readySignal.wait(); - expect(ready).toBe(42); - }); -}); diff --git a/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.ts b/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.ts deleted file mode 100644 index 0512def2b5977..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/utils/ready_signal.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface ReadySignal { - wait(): Promise; - signal(value: T): void; -} - -export function createReadySignal(): ReadySignal { - let resolver: (value: T) => void; - - const promise = new Promise((resolve) => { - resolver = resolve; - }); - - async function wait(): Promise { - return await promise; - } - - function signal(value: T) { - resolver(value); - } - - return { wait, signal }; -} diff --git a/x-pack/plugins/rule_registry/server/event_log/utils/utility_types.ts b/x-pack/plugins/rule_registry/server/event_log/utils/utility_types.ts deleted file mode 100644 index 78e145740da54..0000000000000 --- a/x-pack/plugins/rule_registry/server/event_log/utils/utility_types.ts +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export type DeepWriteable = { -readonly [P in keyof T]: DeepWriteable }; - -export type DeepPartial = { - [P in keyof T]?: T[P] extends Array ? Array> : DeepPartial; -}; diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index cbd8145a44fe7..19257845fabe4 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -8,12 +8,17 @@ import { PluginInitializerContext } from 'src/core/server'; import { RuleRegistryPlugin } from './plugin'; -export * from './config'; export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } from './plugin'; -export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types'; -export { RuleDataPluginService } from './rule_data_plugin_service'; -export { RuleDataClient } from './rule_data_client'; -export { IRuleDataClient } from './rule_data_client/types'; +export type { + RacRequestHandlerContext, + RacApiRequestHandlerContext, + AlertTypeWithExecutor, +} from './types'; + +export * from './config'; +export * from './rule_data_plugin_service'; +export * from './rule_data_client'; + export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data'; export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory'; export { @@ -24,7 +29,6 @@ export { } from './utils/create_lifecycle_executor'; export { createPersistenceRuleTypeFactory } from './utils/create_persistence_rule_type_factory'; export * from './utils/persistence_types'; -export type { AlertTypeWithExecutor } from './types'; export const plugin = (initContext: PluginInitializerContext) => new RuleRegistryPlugin(initContext); diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index ca98254037732..2325e5dd20233 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ + import { PluginInitializerContext, Plugin, @@ -13,30 +14,27 @@ import { CoreStart, IContextProvider, } from 'src/core/server'; -import { SecurityPluginSetup } from '../../security/server'; -import { AlertsClientFactory } from './alert_data_client/alerts_client_factory'; + import { PluginStartContract as AlertingStart } from '../../alerting/server'; -import { RacApiRequestHandlerContext, RacRequestHandlerContext } from './types'; -import { defineRoutes } from './routes'; -import { SpacesPluginStart } from '../../spaces/server'; +import { SecurityPluginSetup } from '../../security/server'; import { RuleRegistryPluginConfig } from './config'; import { RuleDataPluginService } from './rule_data_plugin_service'; -import { EventLogService, IEventLogService } from './event_log'; +import { AlertsClientFactory } from './alert_data_client/alerts_client_factory'; import { AlertsClient } from './alert_data_client/alerts_client'; +import { RacApiRequestHandlerContext, RacRequestHandlerContext } from './types'; +import { defineRoutes } from './routes'; export interface RuleRegistryPluginSetupDependencies { security?: SecurityPluginSetup; } export interface RuleRegistryPluginStartDependencies { - spaces: SpacesPluginStart; alerting: AlertingStart; } export interface RuleRegistryPluginSetupContract { ruleDataService: RuleDataPluginService; - eventLogService: IEventLogService; } export interface RuleRegistryPluginStartContract { @@ -54,7 +52,6 @@ export class RuleRegistryPlugin > { private readonly config: RuleRegistryPluginConfig; private readonly logger: Logger; - private eventLogService: EventLogService | null; private readonly alertsClientFactory: AlertsClientFactory; private ruleDataService: RuleDataPluginService | null; private security: SecurityPluginSetup | undefined; @@ -62,7 +59,6 @@ export class RuleRegistryPlugin constructor(initContext: PluginInitializerContext) { this.config = initContext.config.get(); this.logger = initContext.logger.get(); - this.eventLogService = null; this.ruleDataService = null; this.alertsClientFactory = new AlertsClientFactory(); } @@ -71,7 +67,7 @@ export class RuleRegistryPlugin core: CoreSetup, plugins: RuleRegistryPluginSetupDependencies ): RuleRegistryPluginSetupContract { - const { logger } = this; + const { config, logger } = this; const startDependencies = core.getStartServices().then(([coreStart, pluginStart]) => { return { @@ -82,24 +78,17 @@ export class RuleRegistryPlugin this.security = plugins.security; - const service = new RuleDataPluginService({ - logger: this.logger, - isWriteEnabled: this.config.write.enabled, - index: this.config.index, + this.ruleDataService = new RuleDataPluginService({ + logger, + isWriteEnabled: config.write.enabled, + index: config.index, getClusterClient: async () => { const deps = await startDependencies; return deps.core.elasticsearch.client.asInternalUser; }, }); - service.init().catch((originalError) => { - const error = new Error('Failed installing assets'); - // @ts-ignore - error.stack = originalError.stack; - this.logger.error(error); - }); - - this.ruleDataService = service; + this.ruleDataService.initializeService(); // ALERTS ROUTES const router = core.http.createRouter(); @@ -110,21 +99,7 @@ export class RuleRegistryPlugin defineRoutes(router); - const eventLogService = new EventLogService({ - config: { - indexPrefix: this.config.index, - isWriteEnabled: this.config.write.enabled, - }, - dependencies: { - clusterClient: startDependencies.then((deps) => deps.core.elasticsearch.client), - spacesService: startDependencies.then((deps) => deps.spaces.spacesService), - logger: logger.get('eventLog'), - }, - }); - - this.eventLogService = eventLogService; - - return { ruleDataService: this.ruleDataService, eventLogService }; + return { ruleDataService: this.ruleDataService }; } public start( @@ -165,13 +140,5 @@ export class RuleRegistryPlugin }; }; - public stop() { - const { eventLogService, logger } = this; - - if (eventLogService) { - eventLogService.stop().catch((e) => { - logger.error(e); - }); - } - } + public stop() {} } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts index 94427eee2e2e5..42af95a1c5736 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts @@ -5,178 +5,5 @@ * 2.0. */ -import { ResponseError } from '@elastic/elasticsearch/lib/errors'; -import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; -import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors'; -import { - IRuleDataClient, - RuleDataClientConstructorOptions, - RuleDataReader, - RuleDataWriter, -} from './types'; - -function getNamespacedAlias(options: { alias: string; namespace?: string }) { - return [options.alias, options.namespace].filter(Boolean).join('-'); -} - -export class RuleDataClient implements IRuleDataClient { - constructor(private readonly options: RuleDataClientConstructorOptions) {} - - private async getClusterClient() { - await this.options.ready(); - return await this.options.getClusterClient(); - } - - isWriteEnabled(): boolean { - return this.options.isWriteEnabled; - } - - getReader(options: { namespace?: string } = {}): RuleDataReader { - const index = `${[this.options.alias, options.namespace].filter(Boolean).join('-')}*`; - - return { - search: async (request) => { - const clusterClient = await this.getClusterClient(); - - const { body } = (await clusterClient.search({ - ...request, - index, - })) as { body: any }; - - return body; - }, - getDynamicIndexPattern: async () => { - const clusterClient = await this.getClusterClient(); - const indexPatternsFetcher = new IndexPatternsFetcher(clusterClient); - - try { - const fields = await indexPatternsFetcher.getFieldsForWildcard({ - pattern: index, - }); - - return { - fields, - timeFieldName: '@timestamp', - title: index, - }; - } catch (err) { - if (err.output?.payload?.code === 'no_matching_indices') { - return { - fields: [], - timeFieldName: '@timestamp', - title: index, - }; - } - throw err; - } - }, - }; - } - - getWriter(options: { namespace?: string } = {}): RuleDataWriter { - const { namespace } = options; - const isWriteEnabled = this.isWriteEnabled(); - const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); - - return { - bulk: async (request) => { - if (!isWriteEnabled) { - throw new RuleDataWriteDisabledError(); - } - - const clusterClient = await this.getClusterClient(); - - const requestWithDefaultParameters = { - ...request, - require_alias: true, - index: alias, - }; - - return clusterClient.bulk(requestWithDefaultParameters).then((response) => { - if (response.body.errors) { - if ( - response.body.items.length > 0 && - (response.body.items.every( - (item) => item.index?.error?.type === 'index_not_found_exception' - ) || - response.body.items.every( - (item) => item.index?.error?.type === 'illegal_argument_exception' - )) - ) { - return this.createWriteTargetIfNeeded({ namespace }).then(() => { - return clusterClient.bulk(requestWithDefaultParameters).then((retryResponse) => { - if (retryResponse.body.errors) { - throw new ResponseError(retryResponse); - } - return retryResponse; - }); - }); - } - const error = new ResponseError(response); - throw error; - } - return response; - }); - }, - }; - } - - async createWriteTargetIfNeeded({ namespace }: { namespace?: string }) { - const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); - - const clusterClient = await this.getClusterClient(); - - const { body: indicesExist } = await clusterClient.indices.exists({ - index: `${alias}-*`, - allow_no_indices: false, - }); - - const concreteIndexName = `${alias}-000001`; - - if (!indicesExist) { - try { - await clusterClient.indices.create({ - index: concreteIndexName, - body: { - aliases: { - [alias]: { - is_write_index: true, - }, - }, - }, - }); - } catch (err) { - // If the index already exists and it's the write index for the alias, - // something else created it so suppress the error. If it's not the write - // index, that's bad, throw an error. - if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') { - const { body: existingIndices } = await clusterClient.indices.get({ - index: concreteIndexName, - }); - if (!existingIndices[concreteIndexName]?.aliases?.[alias]?.is_write_index) { - throw Error( - `Attempted to create index: ${concreteIndexName} as the write index for alias: ${alias}, but the index already exists and is not the write index for the alias` - ); - } - } else { - throw err; - } - } - } else { - // If we find indices matching the pattern, then we expect one of them to be the write index for the alias. - // Throw an error if none of them are the write index. - const { body: aliasesResponse } = await clusterClient.indices.getAlias({ - index: `${alias}-*`, - }); - if ( - !Object.entries(aliasesResponse).some( - ([_, aliasesObject]) => aliasesObject.aliases[alias]?.is_write_index - ) - ) { - throw Error( - `Indices matching pattern ${alias}-* exist but none are set as the write index for alias ${alias}` - ); - } - } - } -} +export * from './rule_data_client'; +export * from './types'; diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts similarity index 61% rename from x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts rename to x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts index 24b06439fe573..8aa5f8a6edf19 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts @@ -4,9 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { PublicContract } from '@kbn/utility-types'; -import type { RuleDataClient } from '.'; -import { RuleDataReader, RuleDataWriter } from './types'; + +import { IRuleDataClient, IRuleDataReader, IRuleDataWriter } from './types'; type MockInstances> = { [K in keyof T]: T[K] extends (...args: infer TArgs) => infer TReturn @@ -14,11 +13,9 @@ type MockInstances> = { : never; }; -type RuleDataClientMock = jest.Mocked< - Omit, 'getWriter' | 'getReader'> -> & { - getWriter: (...args: Parameters) => MockInstances; - getReader: (...args: Parameters) => MockInstances; +type RuleDataClientMock = jest.Mocked> & { + getReader: (...args: Parameters) => MockInstances; + getWriter: (...args: Parameters) => MockInstances; }; export function createRuleDataClientMock(): RuleDataClientMock { @@ -27,14 +24,17 @@ export function createRuleDataClientMock(): RuleDataClientMock { const getDynamicIndexPattern = jest.fn(); return { - createWriteTargetIfNeeded: jest.fn(({}) => Promise.resolve()), + indexName: '.alerts-security.alerts', + + isWriteEnabled: jest.fn(() => true), + getReader: jest.fn((_options?: { namespace?: string }) => ({ getDynamicIndexPattern, search, })), + getWriter: jest.fn(() => ({ bulk, })), - isWriteEnabled: jest.fn(() => true), }; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts new file mode 100644 index 0000000000000..bbfa6abdd1a71 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { Either, isLeft } from 'fp-ts/lib/Either'; + +import { ElasticsearchClient } from 'kibana/server'; +import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; + +import { RuleDataWriteDisabledError } from '../rule_data_plugin_service/errors'; +import { IndexInfo } from '../rule_data_plugin_service/index_info'; +import { ResourceInstaller } from '../rule_data_plugin_service/resource_installer'; +import { IRuleDataClient, IRuleDataReader, IRuleDataWriter } from './types'; + +interface ConstructorOptions { + indexInfo: IndexInfo; + resourceInstaller: ResourceInstaller; + isWriteEnabled: boolean; + waitUntilReadyForReading: Promise; + waitUntilReadyForWriting: Promise; +} + +export type WaitResult = Either; + +export class RuleDataClient implements IRuleDataClient { + constructor(private readonly options: ConstructorOptions) {} + + public get indexName(): string { + return this.options.indexInfo.baseName; + } + + public isWriteEnabled(): boolean { + return this.options.isWriteEnabled; + } + + public getReader(options: { namespace?: string } = {}): IRuleDataReader { + const { indexInfo } = this.options; + const indexPattern = indexInfo.getPatternForReading(options.namespace); + + const waitUntilReady = async () => { + const result = await this.options.waitUntilReadyForReading; + if (isLeft(result)) { + throw result.left; + } else { + return result.right; + } + }; + + return { + search: async (request) => { + const clusterClient = await waitUntilReady(); + + const { body } = (await clusterClient.search({ + ...request, + index: indexPattern, + })) as { body: any }; + + return body; + }, + + getDynamicIndexPattern: async () => { + const clusterClient = await waitUntilReady(); + const indexPatternsFetcher = new IndexPatternsFetcher(clusterClient); + + try { + const fields = await indexPatternsFetcher.getFieldsForWildcard({ + pattern: indexPattern, + }); + + return { + fields, + timeFieldName: '@timestamp', + title: indexPattern, + }; + } catch (err) { + if (err.output?.payload?.code === 'no_matching_indices') { + return { + fields: [], + timeFieldName: '@timestamp', + title: indexPattern, + }; + } + throw err; + } + }, + }; + } + + public getWriter(options: { namespace?: string } = {}): IRuleDataWriter { + const { indexInfo, resourceInstaller } = this.options; + + const namespace = options.namespace || 'default'; + const alias = indexInfo.getPrimaryAlias(namespace); + const isWriteEnabled = this.isWriteEnabled(); + + const waitUntilReady = async () => { + const result = await this.options.waitUntilReadyForWriting; + if (isLeft(result)) { + throw result.left; + } else { + return result.right; + } + }; + + return { + bulk: async (request) => { + if (!isWriteEnabled) { + throw new RuleDataWriteDisabledError(); + } + + const clusterClient = await waitUntilReady(); + + const requestWithDefaultParameters = { + ...request, + require_alias: true, + index: alias, + }; + + return clusterClient.bulk(requestWithDefaultParameters).then((response) => { + if (response.body.errors) { + if ( + response.body.items.length > 0 && + (response.body.items.every( + (item) => item.index?.error?.type === 'index_not_found_exception' + ) || + response.body.items.every( + (item) => item.index?.error?.type === 'illegal_argument_exception' + )) + ) { + return resourceInstaller + .installNamespaceLevelResources(indexInfo, namespace) + .then(() => { + return clusterClient.bulk(requestWithDefaultParameters).then((retryResponse) => { + if (retryResponse.body.errors) { + throw new ResponseError(retryResponse); + } + return retryResponse; + }); + }); + } + const error = new ResponseError(response); + throw error; + } + return response; + }); + }, + }; + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index ef09dcc7550a0..979aa7e264848 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -7,19 +7,25 @@ import { ApiResponse } from '@elastic/elasticsearch'; import { BulkRequest, BulkResponse } from '@elastic/elasticsearch/api/types'; -import { ValidFeatureId } from '@kbn/rule-data-utils'; -import { ElasticsearchClient } from 'kibana/server'; -import { FieldDescriptor } from 'src/plugins/data/server'; import { ESSearchRequest, ESSearchResponse } from 'src/core/types/elasticsearch'; +import { FieldDescriptor } from 'src/plugins/data/server'; import { TechnicalRuleDataFieldName } from '../../common/technical_rule_data_field_names'; -export interface RuleDataReader { +export interface IRuleDataClient { + indexName: string; + isWriteEnabled(): boolean; + getReader(options?: { namespace?: string }): IRuleDataReader; + getWriter(options?: { namespace?: string }): IRuleDataWriter; +} + +export interface IRuleDataReader { search( request: TSearchRequest ): Promise< ESSearchResponse>, TSearchRequest> >; + getDynamicIndexPattern( target?: string ): Promise<{ @@ -29,28 +35,6 @@ export interface RuleDataReader { }>; } -export interface RuleDataWriter { +export interface IRuleDataWriter { bulk(request: BulkRequest): Promise>; } - -export interface IRuleDataClient { - getReader(options?: { namespace?: string }): RuleDataReader; - getWriter(options?: { namespace?: string }): RuleDataWriter; - isWriteEnabled(): boolean; - createWriteTargetIfNeeded(options: { namespace?: string }): Promise; -} - -/** - * The purpose of the `feature` param is to force the user to update - * the data structure which contains the mapping of consumers to alerts - * as data indices. The idea is it is typed such that it forces the - * user to go to the code and modify it. At least until a better system - * is put in place or we move the alerts as data client out of rule registry. - */ -export interface RuleDataClientConstructorOptions { - getClusterClient: () => Promise; - isWriteEnabled: boolean; - ready: () => Promise; - alias: string; - feature: ValidFeatureId; -} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts index e10b58a1c60c3..206fabfdd27d6 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts @@ -4,244 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { ClusterPutComponentTemplate } from '@elastic/elasticsearch/api/requestParams'; -import { estypes } from '@elastic/elasticsearch'; -import { ValidFeatureId } from '@kbn/rule-data-utils'; -import { ElasticsearchClient, Logger } from 'kibana/server'; -import { get, isEmpty } from 'lodash'; -import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; -import { - DEFAULT_ILM_POLICY_ID, - ECS_COMPONENT_TEMPLATE_NAME, - TECHNICAL_COMPONENT_TEMPLATE_NAME, -} from '../../common/assets'; -import { ecsComponentTemplate } from '../../common/assets/component_templates/ecs_component_template'; -import { defaultLifecyclePolicy } from '../../common/assets/lifecycle_policies/default_lifecycle_policy'; -import { ClusterPutComponentTemplateBody, PutIndexTemplateRequest } from '../../common/types'; -import { RuleDataClient } from '../rule_data_client'; -import { RuleDataWriteDisabledError } from './errors'; -import { incrementIndexName } from './utils'; - -const BOOTSTRAP_TIMEOUT = 60000; - -export interface RuleDataPluginServiceConstructorOptions { - getClusterClient: () => Promise; - logger: Logger; - isWriteEnabled: boolean; - index: string; -} - -function createSignal() { - let resolver: () => void; - - let ready: boolean = false; - - const promise = new Promise((resolve) => { - resolver = resolve; - }); - - function wait(): Promise { - return promise.then(() => { - ready = true; - }); - } - - function complete() { - resolver(); - } - - return { wait, complete, isReady: () => ready }; -} - -export class RuleDataPluginService { - signal = createSignal(); - - constructor(private readonly options: RuleDataPluginServiceConstructorOptions) {} - - private assertWriteEnabled() { - if (!this.isWriteEnabled()) { - throw new RuleDataWriteDisabledError(); - } - } - - private async getClusterClient() { - return await this.options.getClusterClient(); - } - - async init() { - if (!this.isWriteEnabled()) { - this.options.logger.info('Write is disabled, not installing assets'); - this.signal.complete(); - return; - } - - this.options.logger.info(`Installing assets in namespace ${this.getFullAssetName()}`); - - await this._createOrUpdateLifecyclePolicy({ - policy: this.getFullAssetName(DEFAULT_ILM_POLICY_ID), - body: defaultLifecyclePolicy, - }); - - await this._createOrUpdateComponentTemplate({ - name: this.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - body: technicalComponentTemplate, - }); - - await this._createOrUpdateComponentTemplate({ - name: this.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME), - body: ecsComponentTemplate, - }); - - this.options.logger.info(`Installed all assets`); - - this.signal.complete(); - } - - private async _createOrUpdateComponentTemplate( - template: ClusterPutComponentTemplate - ) { - this.assertWriteEnabled(); - - const clusterClient = await this.getClusterClient(); - this.options.logger.debug(`Installing component template ${template.name}`); - return clusterClient.cluster.putComponentTemplate(template); - } - - private async _createOrUpdateIndexTemplate(template: PutIndexTemplateRequest) { - this.assertWriteEnabled(); - - const clusterClient = await this.getClusterClient(); - this.options.logger.debug(`Installing index template ${template.name}`); - const { body: simulateResponse } = await clusterClient.indices.simulateTemplate(template); - const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings; - - if (isEmpty(mappings)) { - throw new Error( - 'No mappings would be generated for this index, possibly due to failed/misconfigured bootstrapping' - ); - } - return clusterClient.indices.putIndexTemplate(template); - } - - private async _createOrUpdateLifecyclePolicy(policy: estypes.IlmPutLifecycleRequest) { - this.assertWriteEnabled(); - const clusterClient = await this.getClusterClient(); - - this.options.logger.debug(`Installing lifecycle policy ${policy.policy}`); - return clusterClient.ilm.putLifecycle(policy); - } - - private async updateAliasWriteIndexMapping({ index, alias }: { index: string; alias: string }) { - const clusterClient = await this.getClusterClient(); - - const simulatedIndexMapping = await clusterClient.indices.simulateIndexTemplate({ - name: index, - }); - const simulatedMapping = get(simulatedIndexMapping, ['body', 'template', 'mappings']); - try { - await clusterClient.indices.putMapping({ - index, - body: simulatedMapping, - }); - return; - } catch (err) { - if (err.meta?.body?.error?.type !== 'illegal_argument_exception') { - /** - * We skip the rollover if we catch anything except for illegal_argument_exception - that's the error - * returned by ES when the mapping update contains a conflicting field definition (e.g., a field changes types). - * We expect to get that error for some mapping changes we might make, and in those cases, - * we want to continue to rollover the index. Other errors are unexpected. - */ - this.options.logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); - return; - } - const newIndexName = incrementIndexName(index); - if (newIndexName == null) { - this.options.logger.error(`Failed to increment write index name for alias: ${alias}`); - return; - } - try { - await clusterClient.indices.rollover({ - alias, - new_index: newIndexName, - }); - } catch (e) { - /** - * If we catch resource_already_exists_exception, that means that the index has been - * rolled over already — nothing to do for us in this case. - */ - if (e?.meta?.body?.error?.type !== 'resource_already_exists_exception') { - this.options.logger.error(`Failed to rollover index for alias ${alias}: ${e.message}`); - } - } - } - } - - async createOrUpdateComponentTemplate( - template: ClusterPutComponentTemplate - ) { - await this.wait(); - return this._createOrUpdateComponentTemplate(template); - } - - async createOrUpdateIndexTemplate(template: PutIndexTemplateRequest) { - await this.wait(); - return this._createOrUpdateIndexTemplate(template); - } - - async createOrUpdateLifecyclePolicy(policy: estypes.IlmPutLifecycleRequest) { - await this.wait(); - return this._createOrUpdateLifecyclePolicy(policy); - } - - async updateIndexMappingsMatchingPattern(pattern: string) { - await this.wait(); - const clusterClient = await this.getClusterClient(); - const { body: aliasesResponse } = await clusterClient.indices.getAlias({ index: pattern }); - const writeIndicesAndAliases: Array<{ index: string; alias: string }> = []; - Object.entries(aliasesResponse).forEach(([index, aliases]) => { - Object.entries(aliases.aliases).forEach(([aliasName, aliasProperties]) => { - if (aliasProperties.is_write_index) { - writeIndicesAndAliases.push({ index, alias: aliasName }); - } - }); - }); - await Promise.all( - writeIndicesAndAliases.map((indexAndAlias) => - this.updateAliasWriteIndexMapping(indexAndAlias) - ) - ); - } - - isReady() { - return this.signal.isReady(); - } - - wait() { - return Promise.race([ - this.signal.wait(), - new Promise((resolve, reject) => { - setTimeout(reject, BOOTSTRAP_TIMEOUT); - }), - ]); - } - - isWriteEnabled(): boolean { - return this.options.isWriteEnabled; - } - - getFullAssetName(assetName?: string) { - return [this.options.index, assetName].filter(Boolean).join('-'); - } - - getRuleDataClient(feature: ValidFeatureId, alias: string, initialize: () => Promise) { - return new RuleDataClient({ - alias, - feature, - getClusterClient: () => this.getClusterClient(), - isWriteEnabled: this.isWriteEnabled(), - ready: initialize, - }); - } -} +export * from './rule_data_plugin_service'; +export * from './index_options'; +export * from './errors'; diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts new file mode 100644 index 0000000000000..4e64ea025a27a --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IndexOptions } from './index_options'; +import { joinWithDash } from './utils'; + +interface ConstructorOptions { + /** + * Prepends a relative resource name (defined in the code) with + * a full resource prefix, which starts with '.alerts' and can + * optionally include a user-defined part in it. + * @example 'security.alerts' => '.alerts-security.alerts' + */ + getResourceName(relativeName: string): string; + + /** + * Options provided by the plugin/solution defining the index. + */ + indexOptions: IndexOptions; +} + +/** + * Internal info used by the index bootstrapping logic, reader and writer. + * Should not be exposed to clients of the library. + * + * Names returned by methods of this class should be used in Elasticsearch APIs. + */ +export class IndexInfo { + constructor(options: ConstructorOptions) { + const { getResourceName, indexOptions } = options; + const { registrationContext, dataset } = indexOptions; + + this.indexOptions = indexOptions; + this.baseName = getResourceName(`${registrationContext}.${dataset}`); + this.basePattern = joinWithDash(this.baseName, '*'); + } + + /** + * Options provided by the plugin/solution defining the index. + */ + public readonly indexOptions: IndexOptions; + + /** + * Base index name, prefixed with the full resource prefix. + * @example '.alerts-security.alerts' + */ + public readonly baseName: string; + + /** + * Base index pattern. Includes all namespaces of this index. + * @example '.alerts-security.alerts-*' + */ + public readonly basePattern: string; + + /** + * Primary index alias. Includes a namespace. + * Used as a write target when writing documents to the index. + * @example '.alerts-security.alerts-default' + */ + public getPrimaryAlias(namespace: string): string { + return joinWithDash(this.baseName, namespace); + } + + /** + * Index pattern based on the primary alias. + * @example '.alerts-security.alerts-default-*' + */ + public getPrimaryAliasPattern(namespace: string): string { + return joinWithDash(this.baseName, namespace, '*'); + } + + /** + * Optional secondary alias that can be applied to concrete indices in + * addition to the primary one. + * @example '.siem-signals-default', null + */ + public getSecondaryAlias(namespace: string): string | null { + const { secondaryAlias } = this.indexOptions; + return secondaryAlias ? joinWithDash(secondaryAlias, namespace) : null; + } + + /** + * Index pattern that should be used when reading documents from the index. + * Can include or exclude the namespace. + * + * IMPORTANT: The namespace is user-defined in general. Because of that, when + * reading data from the index, we want to do this by default: + * - pass namespace = undefined + * - search over all the namespaces + * - include nested registration contexts eagerly + * - e.g. if baseName='.alerts-observability', include '.alerts-observability.apm' + * + * @example '.alerts-security.alerts-default*', '.alerts-security.alerts*' + */ + public getPatternForReading(namespace?: string): string { + return `${joinWithDash(this.baseName, namespace)}*`; + } + + /** + * Name of the initial concrete index, with the namespace and the ILM suffix. + * @example '.alerts-security.alerts-default-000001' + */ + public getConcreteIndexInitialName(namespace: string): string { + return joinWithDash(this.baseName, namespace, '000001'); + } + + /** + * Name of the custom ILM policy (if it's provided by the plugin/solution). + * Specific to the index. Shared between all namespaces of the index. + * @example '.alerts-security.alerts-policy' + */ + public getIlmPolicyName(): string { + return joinWithDash(this.baseName, 'policy'); + } + + /** + * Full name of a component template. + * @example '.alerts-security.alerts-mappings' + */ + public getComponentTemplateName(relativeName: string): string { + return joinWithDash(this.baseName, relativeName); + } + + /** + * Full name of the index template. Each namespace gets its own template. + * @example '.alerts-security.alerts-default-index-template' + */ + public getIndexTemplateName(namespace: string): string { + return joinWithDash(this.baseName, namespace, 'index-template'); + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts new file mode 100644 index 0000000000000..0f9486a068c24 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts @@ -0,0 +1,153 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { ValidFeatureId } from '@kbn/rule-data-utils'; + +/** + * Options that a plugin/solution provides to rule_registry in order to + * define and initialize an index for alerts-as-data. + * + * IMPORTANT: All names provided in these options are relative. For example: + * - component template refs will be 'ecs-mappings', not '.alerts-ecs-mappings' + * - component template names will be 'mappings', not '.alerts-security.alerts-mappings' + * - etc + */ +export interface IndexOptions { + /** + * ID of the Kibana feature associated with the index. + * Used by alerts-as-data RBAC. + * + * Note from @dhurley14 + * The purpose of the `feature` param is to force the user to update + * the data structure which contains the mapping of consumers to alerts + * as data indices. The idea is it is typed such that it forces the + * user to go to the code and modify it. At least until a better system + * is put in place or we move the alerts as data client out of rule registry. + * + * @example 'siem', 'logs', 'apm' + */ + feature: ValidFeatureId; + + /** + * Registration context which defines a solution or an app within a solution. + * @example 'security', 'observability', 'observability.logs' + */ + registrationContext: string; + + /** + * Dataset suffix. Restricted to a few values. + * @example 'alerts', 'events' + */ + dataset: Dataset; + + /** + * A list of references to external component templates. Those can be + * the common ones shared between all solutions, or special ones + * shared between some of them. + * + * IMPORTANT: These names should be relative. + * - correct: 'my-mappings' + * - incorrect: '.alerts-my-mappings' + * + * @example ['ecs-mappings'] + */ + componentTemplateRefs: string[]; + + /** + * Own component templates specified for the index by the plugin/solution + * defining this index. + * + * IMPORTANT: Order matters. This order is used by Elasticsearch to set + * priorities when merging the same field names defined in 2+ templates. + * + * IMPORTANT: Component template names should be relative. + * - correct: 'mappings' + * - incorrect: 'security.alerts-mappings' + * - incorrect: '.alerts-security.alerts-mappings' + */ + componentTemplates: ComponentTemplateOptions[]; + + /** + * Additional properties for the namespaced index template. + */ + indexTemplate: IndexTemplateOptions; + + /** + * Optional custom ILM policy for the index. + * NOTE: this policy will be shared between all namespaces of the index. + */ + ilmPolicy?: IlmPolicyOptions; + + /** + * Optional secondary alias that will be applied to concrete indices in + * addition to the primary one '.alerts-{reg. context}.{dataset}-{namespace}' + * + * IMPORTANT: It should not include the namespace. It will be added + * automatically. + * - correct: '.siem-signals' + * - incorrect: '.siem-signals-default' + * + * @example '.siem-signals', undefined + */ + secondaryAlias?: string; +} + +/** + * Dataset suffix restricted to a few values. All alerts-as-data indices + * are designed to contain only documents of these "kinds". + */ +export enum Dataset { + alerts = 'alerts', + events = 'events', +} + +export type Settings = estypes.IndicesIndexSettings; +export type Mappings = estypes.MappingTypeMapping; +export type Version = estypes.VersionNumber; +export type Meta = estypes.Metadata; + +/** + * When initializing an index, a plugin/solution can break mappings and settings + * down into several component templates. Some of their properties can be + * defined by the plugin/solution via these options. + * + * https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-component-template.html + */ +export interface ComponentTemplateOptions { + name: string; + version: Version; // TODO: encapsulate versioning (base on Kibana version) + mappings?: Mappings; + settings?: Settings; + _meta?: Meta; +} + +/** + * When initializing an index, a plugin/solution can provide some optional + * properties which will be included into the index template. + * + * Note that: + * - each index namespace will get its own index template + * - the template will be created by the library + * - most of its properties will be set by the library + * - you can inject some of them via these options + * + * https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-template.html + * https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html + */ +export interface IndexTemplateOptions { + version: Version; // TODO: encapsulate versioning (base on Kibana version) + _meta?: Meta; +} + +/** + * When initializing an index, a plugin/solution can provide a custom + * ILM policy that will be applied to concrete indices of this index. + * + * Note that policy will be shared between all namespaces of the index. + */ +export type IlmPolicyOptions = Omit; diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts new file mode 100644 index 0000000000000..7ad5926d53d08 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -0,0 +1,421 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isEmpty } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; + +import { ElasticsearchClient, Logger } from 'kibana/server'; + +import { + DEFAULT_ILM_POLICY_ID, + ECS_COMPONENT_TEMPLATE_NAME, + TECHNICAL_COMPONENT_TEMPLATE_NAME, +} from '../../common/assets'; +import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; +import { ecsComponentTemplate } from '../../common/assets/component_templates/ecs_component_template'; +import { defaultLifecyclePolicy } from '../../common/assets/lifecycle_policies/default_lifecycle_policy'; + +import { IndexInfo } from './index_info'; +import { incrementIndexName } from './utils'; + +const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes + +interface ConstructorOptions { + getResourceName(relativeName: string): string; + getClusterClient: () => Promise; + logger: Logger; + isWriteEnabled: boolean; +} + +export class ResourceInstaller { + constructor(private readonly options: ConstructorOptions) {} + + private async installWithTimeout( + resources: string, + installer: () => Promise + ): Promise { + try { + const installResources = async (): Promise => { + const { logger, isWriteEnabled } = this.options; + + if (!isWriteEnabled) { + logger.info(`Write is disabled; not installing ${resources}`); + return; + } + + logger.info(`Installing ${resources}`); + await installer(); + logger.info(`Installed ${resources}`); + }; + + const throwTimeoutException = (): Promise => { + return new Promise((resolve, reject) => { + setTimeout(() => { + const msg = `Timeout: it took more than ${INSTALLATION_TIMEOUT}ms`; + reject(new Error(msg)); + }, INSTALLATION_TIMEOUT); + }); + }; + + await Promise.race([installResources(), throwTimeoutException()]); + } catch (e) { + this.options.logger.error(e); + + const reason = e?.message || 'Unknown reason'; + throw new Error(`Failure installing ${resources}. ${reason}`); + } + } + + // ----------------------------------------------------------------------------------------------- + // Common resources + + /** + * Installs common, library-level resources shared between all indices: + * - default ILM policy + * - component template containing technical fields + * - component template containing all standard ECS fields + */ + public async installCommonResources(): Promise { + await this.installWithTimeout('common resources shared between all indices', async () => { + const { getResourceName } = this.options; + + // We can install them in parallel + await Promise.all([ + this.createOrUpdateLifecyclePolicy({ + policy: getResourceName(DEFAULT_ILM_POLICY_ID), + body: defaultLifecyclePolicy, + }), + + this.createOrUpdateComponentTemplate({ + name: getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + body: technicalComponentTemplate, + }), + + this.createOrUpdateComponentTemplate({ + name: getResourceName(ECS_COMPONENT_TEMPLATE_NAME), + body: ecsComponentTemplate, + }), + ]); + }); + } + + // ----------------------------------------------------------------------------------------------- + // Index-level resources + + /** + * Installs index-level resources shared between all namespaces of this index: + * - custom ILM policy if it was provided + * - component templates + * - attempts to update mappings of existing concrete indices + */ + public async installIndexLevelResources(indexInfo: IndexInfo): Promise { + await this.installWithTimeout(`resources for index ${indexInfo.baseName}`, async () => { + const { componentTemplates, ilmPolicy } = indexInfo.indexOptions; + + if (ilmPolicy != null) { + await this.createOrUpdateLifecyclePolicy({ + policy: indexInfo.getIlmPolicyName(), + body: { policy: ilmPolicy }, + }); + } + + await Promise.all( + componentTemplates.map(async (ct) => { + await this.createOrUpdateComponentTemplate({ + name: indexInfo.getComponentTemplateName(ct.name), + body: { + template: { + settings: ct.settings ?? {}, + mappings: ct.mappings, + }, + version: ct.version, + _meta: ct._meta, + }, + }); + }) + ); + + // TODO: Update all existing namespaced index templates matching this index' base name + + await this.updateIndexMappings(indexInfo); + }); + } + + private async updateIndexMappings(indexInfo: IndexInfo) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + logger.debug(`Updating mappings of existing concrete indices for ${indexInfo.baseName}`); + + const { body: aliasesResponse } = await clusterClient.indices.getAlias({ + index: indexInfo.basePattern, + }); + + const writeIndicesAndAliases = Object.entries(aliasesResponse).flatMap(([index, { aliases }]) => + Object.entries(aliases) + .filter(([, aliasProperties]) => aliasProperties.is_write_index) + .map(([aliasName]) => ({ index, alias: aliasName })) + ); + + await Promise.all( + writeIndicesAndAliases.map((indexAndAlias) => + this.updateAliasWriteIndexMapping(indexAndAlias) + ) + ); + } + + private async updateAliasWriteIndexMapping({ index, alias }: { index: string; alias: string }) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + const simulatedIndexMapping = await clusterClient.indices.simulateIndexTemplate({ + name: index, + }); + const simulatedMapping = get(simulatedIndexMapping, ['body', 'template', 'mappings']); + + try { + await clusterClient.indices.putMapping({ + index, + body: simulatedMapping, + }); + return; + } catch (err) { + if (err.meta?.body?.error?.type !== 'illegal_argument_exception') { + /** + * We skip the rollover if we catch anything except for illegal_argument_exception - that's the error + * returned by ES when the mapping update contains a conflicting field definition (e.g., a field changes types). + * We expect to get that error for some mapping changes we might make, and in those cases, + * we want to continue to rollover the index. Other errors are unexpected. + */ + logger.error(`Failed to PUT mapping for alias ${alias}: ${err.message}`); + return; + } + const newIndexName = incrementIndexName(index); + if (newIndexName == null) { + logger.error(`Failed to increment write index name for alias: ${alias}`); + return; + } + try { + await clusterClient.indices.rollover({ + alias, + new_index: newIndexName, + }); + } catch (e) { + /** + * If we catch resource_already_exists_exception, that means that the index has been + * rolled over already — nothing to do for us in this case. + */ + if (e?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Failed to rollover index for alias ${alias}: ${e.message}`); + } + } + } + } + + // ----------------------------------------------------------------------------------------------- + // Namespace-level resources + + /** + * Installs resources tied to concrete namespace of an index: + * - namespaced index template + * - concrete index (write target) if it doesn't exist + */ + public async installNamespaceLevelResources( + indexInfo: IndexInfo, + namespace: string + ): Promise { + await this.createWriteTargetIfNeeded(indexInfo, namespace); + } + + private async createWriteTargetIfNeeded(indexInfo: IndexInfo, namespace: string) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace); + const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace); + const initialIndexName = indexInfo.getConcreteIndexInitialName(namespace); + + logger.debug(`Creating write target for ${primaryNamespacedAlias}`); + + const { body: indicesExist } = await clusterClient.indices.exists({ + index: primaryNamespacedPattern, + allow_no_indices: false, + }); + + if (!indicesExist) { + await this.installNamespacedIndexTemplate(indexInfo, namespace); + + try { + await clusterClient.indices.create({ + index: initialIndexName, + body: { + aliases: { + [primaryNamespacedAlias]: { + is_write_index: true, + }, + }, + }, + }); + } catch (err) { + // If the index already exists and it's the write index for the alias, + // something else created it so suppress the error. If it's not the write + // index, that's bad, throw an error. + if (err?.meta?.body?.error?.type === 'resource_already_exists_exception') { + const { body: existingIndices } = await clusterClient.indices.get({ + index: initialIndexName, + }); + if ( + !existingIndices[initialIndexName]?.aliases?.[primaryNamespacedAlias]?.is_write_index + ) { + throw Error( + `Attempted to create index: ${initialIndexName} as the write index for alias: ${primaryNamespacedAlias}, but the index already exists and is not the write index for the alias` + ); + } + } else { + throw err; + } + } + } else { + // If we find indices matching the pattern, then we expect one of them to be the write index for the alias. + // Throw an error if none of them are the write index. + const { body: aliasesResponse } = await clusterClient.indices.getAlias({ + index: primaryNamespacedPattern, + }); + if ( + !Object.entries(aliasesResponse).some( + ([_, aliasesObject]) => aliasesObject.aliases[primaryNamespacedAlias]?.is_write_index + ) + ) { + throw Error( + `Indices matching pattern ${primaryNamespacedPattern} exist but none are set as the write index for alias ${primaryNamespacedAlias}` + ); + } + } + } + + private async installNamespacedIndexTemplate(indexInfo: IndexInfo, namespace: string) { + const { logger, getResourceName } = this.options; + const { + componentTemplateRefs, + componentTemplates, + indexTemplate, + ilmPolicy, + } = indexInfo.indexOptions; + + const primaryNamespacedAlias = indexInfo.getPrimaryAlias(namespace); + const primaryNamespacedPattern = indexInfo.getPrimaryAliasPattern(namespace); + const secondaryNamespacedAlias = indexInfo.getSecondaryAlias(namespace); + + logger.debug(`Installing index template for ${primaryNamespacedAlias}`); + + const technicalComponentNames = [getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME)]; + const referencedComponentNames = componentTemplateRefs.map((ref) => getResourceName(ref)); + const ownComponentNames = componentTemplates.map((template) => + indexInfo.getComponentTemplateName(template.name) + ); + const ilmPolicyName = ilmPolicy + ? indexInfo.getIlmPolicyName() + : getResourceName(DEFAULT_ILM_POLICY_ID); + + // TODO: need a way to update this template if/when we decide to make changes to the + // built in index template. Probably do it as part of updateIndexMappingsForAsset? + // (Before upgrading any indices, find and upgrade all namespaced index templates - component templates + // will already have been upgraded by solutions or rule registry, in the case of technical/ECS templates) + // With the current structure, it's tricky because the index template creation + // depends on both the namespace and secondary alias, both of which are not currently available + // to updateIndexMappingsForAsset. We can make the secondary alias available since + // it's known at plugin startup time, but + // the namespace values can really only come from the existing templates that we're trying to update + // - maybe we want to store the namespace as a _meta field on the index template for easy retrieval + await this.createOrUpdateIndexTemplate({ + name: indexInfo.getIndexTemplateName(namespace), + body: { + index_patterns: [primaryNamespacedPattern], + + // Order matters: + // - first go external component templates referenced by this index (e.g. the common full ECS template) + // - then we include own component templates registered with this index + // - finally, we include technical component templates to make sure the index gets all the + // mappings and settings required by all Kibana plugins using rule registry to work properly + composed_of: [ + ...referencedComponentNames, + ...ownComponentNames, + ...technicalComponentNames, + ], + + template: { + settings: { + 'index.lifecycle': { + name: ilmPolicyName, + // TODO: fix the types in the ES package, they don't include rollover_alias??? + // @ts-expect-error + rollover_alias: primaryNamespacedAlias, + }, + }, + aliases: + secondaryNamespacedAlias != null + ? { + [secondaryNamespacedAlias]: { + is_write_index: false, + }, + } + : undefined, + }, + + _meta: { + ...indexTemplate._meta, + namespace, + }, + + version: indexTemplate.version, + + // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace + // then newly created indices will use the matching template with the *longest* namespace + priority: namespace.length, + }, + }); + } + + // ----------------------------------------------------------------------------------------------- + // Helpers + + private async createOrUpdateLifecyclePolicy(policy: estypes.IlmPutLifecycleRequest) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + logger.debug(`Installing lifecycle policy ${policy.policy}`); + return clusterClient.ilm.putLifecycle(policy); + } + + private async createOrUpdateComponentTemplate( + template: estypes.ClusterPutComponentTemplateRequest + ) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + logger.debug(`Installing component template ${template.name}`); + return clusterClient.cluster.putComponentTemplate(template); + } + + private async createOrUpdateIndexTemplate(template: estypes.IndicesPutIndexTemplateRequest) { + const { logger, getClusterClient } = this.options; + const clusterClient = await getClusterClient(); + + logger.debug(`Installing index template ${template.name}`); + + const { body: simulateResponse } = await clusterClient.indices.simulateTemplate(template); + const mappings: estypes.MappingTypeMapping = simulateResponse.template.mappings; + + if (isEmpty(mappings)) { + throw new Error( + 'No mappings would be generated for this index, possibly due to failed/misconfigured bootstrapping' + ); + } + + return clusterClient.indices.putIndexTemplate(template); + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts index 348c0fc6b1cfc..467d6816d0443 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.mock.ts @@ -6,22 +6,17 @@ */ import type { PublicMethodsOf } from '@kbn/utility-types'; -import { RuleDataPluginService } from './'; +import { RuleDataPluginService } from './rule_data_plugin_service'; type Schema = PublicMethodsOf; const createRuleDataPluginService = () => { const mocked: jest.Mocked = { - init: jest.fn(), - isReady: jest.fn(), - wait: jest.fn(), + getResourcePrefix: jest.fn(), + getResourceName: jest.fn(), isWriteEnabled: jest.fn(), - getFullAssetName: jest.fn(), - createOrUpdateComponentTemplate: jest.fn(), - createOrUpdateIndexTemplate: jest.fn(), - createOrUpdateLifecyclePolicy: jest.fn(), - getRuleDataClient: jest.fn(), - updateIndexMappingsMatchingPattern: jest.fn(), + initializeService: jest.fn(), + initializeIndex: jest.fn(), }; return mocked; }; diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts new file mode 100644 index 0000000000000..b95effd4801b9 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Either, isLeft, left, right } from 'fp-ts/lib/Either'; + +import { ElasticsearchClient, Logger } from 'kibana/server'; + +import { IRuleDataClient, RuleDataClient, WaitResult } from '../rule_data_client'; +import { IndexInfo } from './index_info'; +import { IndexOptions } from './index_options'; +import { ResourceInstaller } from './resource_installer'; +import { joinWithDash } from './utils'; + +interface ConstructorOptions { + getClusterClient: () => Promise; + logger: Logger; + isWriteEnabled: boolean; + index: string; +} + +/** + * A service for creating and using Elasticsearch indices for alerts-as-data. + */ +export class RuleDataPluginService { + private readonly resourceInstaller: ResourceInstaller; + private installCommonResources: Promise>; + private isInitialized: boolean; + + constructor(private readonly options: ConstructorOptions) { + this.resourceInstaller = new ResourceInstaller({ + getResourceName: (name) => this.getResourceName(name), + getClusterClient: options.getClusterClient, + logger: options.logger, + isWriteEnabled: options.isWriteEnabled, + }); + + this.installCommonResources = Promise.resolve(right('ok')); + this.isInitialized = false; + } + + /** + * Returns a full resource prefix. + * - it's '.alerts' by default + * - it can be adjusted by the user via Kibana config + */ + public getResourcePrefix(): string { + // TODO: https://github.com/elastic/kibana/issues/106432 + return this.options.index; + } + + /** + * Prepends a relative resource name with a full resource prefix, which + * starts with '.alerts' and can optionally include a user-defined part in it. + * @returns Full name of the resource. + * @example 'security.alerts' => '.alerts-security.alerts' + */ + public getResourceName(relativeName: string): string { + return joinWithDash(this.getResourcePrefix(), relativeName); + } + + /** + * If write is enabled, everything works as usual. + * If it's disabled, writing to all alerts-as-data indices will be disabled, + * and also Elasticsearch resources associated with the indices will not be + * installed. + */ + public isWriteEnabled(): boolean { + return this.options.isWriteEnabled; + } + + /** + * Installs common Elasticsearch resources used by all alerts-as-data indices. + */ + public initializeService(): void { + // Run the installation of common resources and handle exceptions. + this.installCommonResources = this.resourceInstaller + .installCommonResources() + .then(() => right('ok' as const)) + .catch((e) => { + this.options.logger.error(e); + return left(e); // propagates it to the index initialization phase + }); + + this.isInitialized = true; + } + + /** + * Initializes alerts-as-data index and starts index bootstrapping right away. + * @param indexOptions Index parameters: names and resources. + * @returns Client for reading and writing data to this index. + */ + public initializeIndex(indexOptions: IndexOptions): IRuleDataClient { + if (!this.isInitialized) { + throw new Error( + 'Rule data service is not initialized. Make sure to call initializeService() in the rule registry plugin setup phase' + ); + } + + const indexInfo = new IndexInfo({ + getResourceName: (name) => this.getResourceName(name), + indexOptions, + }); + + const waitUntilClusterClientAvailable = async (): Promise => { + try { + const clusterClient = await this.options.getClusterClient(); + return right(clusterClient); + } catch (e) { + this.options.logger.error(e); + return left(e); + } + }; + + const waitUntilIndexResourcesInstalled = async (): Promise => { + try { + const result = await this.installCommonResources; + if (isLeft(result)) { + return result; + } + + await this.resourceInstaller.installIndexLevelResources(indexInfo); + + const clusterClient = await this.options.getClusterClient(); + return right(clusterClient); + } catch (e) { + this.options.logger.error(e); + return left(e); + } + }; + + // Start initialization now, including installation of index resources. + // Let's unblock read operations since installation can take quite some time. + // Write operations will have to wait, of course. + // NOTE: these promises cannot reject, otherwise it will lead to an + // unhandled promise rejection shutting down Kibana process. + const waitUntilReadyForReading = waitUntilClusterClientAvailable(); + const waitUntilReadyForWriting = waitUntilIndexResourcesInstalled(); + + return new RuleDataClient({ + indexInfo, + resourceInstaller: this.resourceInstaller, + isWriteEnabled: this.isWriteEnabled(), + waitUntilReadyForReading, + waitUntilReadyForWriting, + }); + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts index aaab338ff858a..10f2f546046b7 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/utils.ts @@ -5,11 +5,19 @@ * 2.0. */ -export function incrementIndexName(oldIndex: string) { +export const incrementIndexName = (oldIndex: string) => { const baseIndexString = oldIndex.slice(0, -6); const newIndexNumber = Number(oldIndex.slice(-6)) + 1; if (isNaN(newIndexNumber)) { return undefined; } return baseIndexString + String(newIndexNumber).padStart(6, '0'); -} +}; + +export const joinWith = (separator: string) => ( + ...items: Array +): string => { + return items.filter(Boolean).map(String).join(separator); +}; + +export const joinWithDash = joinWith('-'); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts index efcc56a1b9511..236f3b41d689d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -27,7 +27,7 @@ import { ALERT_RULE_CONSUMER, SPACE_IDS, } from '../../common/technical_rule_data_field_names'; -import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock'; +import { createRuleDataClientMock } from '../rule_data_client/rule_data_client.mock'; import { createLifecycleExecutor } from './create_lifecycle_executor'; describe('createLifecycleExecutor', () => { diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 7a00457f2c4e1..e6db167d8c6b6 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -35,7 +35,7 @@ import { TIMESTAMP, SPACE_IDS, } from '../../common/technical_rule_data_field_names'; -import { RuleDataClient } from '../rule_data_client'; +import { IRuleDataClient } from '../rule_data_client'; import { AlertExecutorOptionsWithExtraServices } from '../types'; import { getRuleData } from './get_rule_executor_data'; @@ -102,7 +102,7 @@ export type WrappedLifecycleRuleState = AlertTypeS export const createLifecycleExecutor = ( logger: Logger, - ruleDataClient: PublicContract + ruleDataClient: PublicContract ) => < Params extends AlertTypeParams = never, State extends AlertTypeState = never, diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 3469187122127..962a37a2991b0 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -10,7 +10,7 @@ import { ALERT_DURATION, ALERT_STATUS, ALERT_UUID } from '@kbn/rule-data-utils'; import { loggerMock } from '@kbn/logging/target/mocks'; import { castArray, omit, mapValues } from 'lodash'; import { RuleDataClient } from '../rule_data_client'; -import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock'; +import { createRuleDataClientMock } from '../rule_data_client/rule_data_client.mock'; import { createLifecycleRuleTypeFactory } from './create_lifecycle_rule_type_factory'; type RuleTestHelpers = ReturnType; diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts index 21e95fbefe4e2..9a809efdb5789 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts @@ -5,7 +5,7 @@ * 2.0. */ import { Logger } from '@kbn/logging'; -import { RuleDataClient } from '..'; +import { IRuleDataClient } from '../rule_data_client'; import { AlertInstanceContext, AlertInstanceState, @@ -20,7 +20,7 @@ export const createLifecycleRuleTypeFactory = ({ ruleDataClient, }: { logger: Logger; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; }) => < TParams extends AlertTypeParams, TAlertInstanceContext extends AlertInstanceContext, diff --git a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts index ea84001dbffd6..11607909a2e0f 100644 --- a/x-pack/plugins/rule_registry/server/utils/persistence_types.ts +++ b/x-pack/plugins/rule_registry/server/utils/persistence_types.ts @@ -15,7 +15,7 @@ import { AlertTypeParams, AlertTypeState, } from '../../../alerting/server'; -import { RuleDataClient } from '../rule_data_client'; +import { IRuleDataClient } from '../rule_data_client'; import { AlertTypeWithExecutor } from '../types'; export type PersistenceAlertService< @@ -38,7 +38,7 @@ export interface PersistenceServices < TState extends AlertTypeState, diff --git a/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts b/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts index 7943c3ad4f35a..0bee5bdf53a69 100644 --- a/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts +++ b/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts @@ -6,10 +6,10 @@ */ import { AlertInstanceContext, AlertTypeParams, AlertTypeState } from '../../../alerting/common'; -import { RuleDataClient } from '../rule_data_client'; +import { IRuleDataClient } from '../rule_data_client'; import { AlertTypeWithExecutor } from '../types'; -export const withRuleDataClientFactory = (ruleDataClient: RuleDataClient) => < +export const withRuleDataClientFactory = (ruleDataClient: IRuleDataClient) => < TState extends AlertTypeState, TParams extends AlertTypeParams, TAlertInstanceContext extends AlertInstanceContext, @@ -19,13 +19,13 @@ export const withRuleDataClientFactory = (ruleDataClient: RuleDataClient) => < TState, TParams, TAlertInstanceContext, - TServices & { ruleDataClient: RuleDataClient } + TServices & { ruleDataClient: IRuleDataClient } > ): AlertTypeWithExecutor< TState, TParams, TAlertInstanceContext, - TServices & { ruleDataClient: RuleDataClient } + TServices & { ruleDataClient: IRuleDataClient } > => { return { ...type, diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json index 769c9f81f8ce9..384ffa0ee3428 100644 --- a/x-pack/plugins/rule_registry/tsconfig.json +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -19,7 +19,6 @@ { "path": "../../../src/plugins/data/tsconfig.json" }, { "path": "../alerting/tsconfig.json" }, { "path": "../security/tsconfig.json" }, - { "path": "../spaces/tsconfig.json" }, { "path": "../triggers_actions_ui/tsconfig.json" } ] } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index c6635eec520b2..7dad03ed7e14e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -110,7 +110,7 @@ export const createDetectionIndex = async ( if (!policyExists) { await setPolicy(esClient, index, signalsPolicy); } - const aadIndexAliasName = `${ruleDataService.getFullAssetName('security.alerts')}-${spaceId}`; + const aadIndexAliasName = ruleDataService.getResourceName(`security.alerts-${spaceId}`); if (await templateNeedsUpdate({ alias: index, esClient })) { await esClient.indices.putIndexTemplate({ name: index, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts index 6ada1e705a852..b7f32b82cc767 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/create_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants'; import { SetupPlugins } from '../../../../plugin'; @@ -25,7 +25,7 @@ import { convertCreateAPIToInternalSchema } from '../../schemas/rule_converters' export const createRulesRoute = ( router: SecuritySolutionPluginRouter, ml: SetupPlugins['ml'], - ruleDataClient?: RuleDataClient | null // TODO: Use this for RAC (otherwise delete it) + ruleDataClient?: IRuleDataClient | null // TODO: Use this for RAC (otherwise delete it) ): void => { router.post( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts index 73d541802f055..2cee8301a05ff 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/delete_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { queryRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/query_rules_type_dependents'; import { queryRulesSchema, @@ -23,7 +23,7 @@ import { readRules } from '../../rules/read_rules'; export const deleteRulesRoute = ( router: SecuritySolutionPluginRouter, - ruleDataClient?: RuleDataClient | null + ruleDataClient?: IRuleDataClient | null ) => { router.delete( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts index f0483c935f71c..4a464c19f5b97 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/find_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { findRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/find_rules_type_dependents'; import { findRulesSchema, @@ -22,7 +22,7 @@ import { getBulkRuleActionsSavedObject } from '../../rule_actions/get_bulk_rule_ export const findRulesRoute = ( router: SecuritySolutionPluginRouter, - ruleDataClient?: RuleDataClient | null + ruleDataClient?: IRuleDataClient | null ) => { router.get( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts index 45217fbd5e62c..1efc9c93b08d2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/patch_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { RuleAlertAction } from '../../../../../common/detection_engine/types'; import { patchRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/patch_rules_type_dependents'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -31,7 +31,7 @@ import { PartialFilter } from '../../types'; export const patchRulesRoute = ( router: SecuritySolutionPluginRouter, ml: SetupPlugins['ml'], - ruleDataClient?: RuleDataClient | null + ruleDataClient?: IRuleDataClient | null ) => { router.patch( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts index fc290190d86ee..6d5e63b2a0588 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/read_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { queryRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/query_rules_type_dependents'; import { queryRulesSchema, @@ -24,7 +24,7 @@ import { RuleExecutionStatus } from '../../../../../common/detection_engine/sche export const readRulesRoute = ( router: SecuritySolutionPluginRouter, - ruleDataClient?: RuleDataClient | null + ruleDataClient?: IRuleDataClient | null ) => { router.get( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts index 23449227f6c70..368b02fdb1e94 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/update_rules_route.ts @@ -6,7 +6,7 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; +import { IRuleDataClient } from '../../../../../../rule_registry/server'; import { updateRulesSchema } from '../../../../../common/detection_engine/schemas/request'; import { updateRuleValidateTypeDependents } from '../../../../../common/detection_engine/schemas/request/update_rules_type_dependents'; import type { SecuritySolutionPluginRouter } from '../../../../types'; @@ -25,7 +25,7 @@ import { buildRouteValidation } from '../../../../utils/build_validation/route_v export const updateRulesRoute = ( router: SecuritySolutionPluginRouter, ml: SetupPlugins['ml'], - ruleDataClient?: RuleDataClient | null + ruleDataClient?: IRuleDataClient | null ) => { router.put( { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_execution_log_bootstrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_execution_log_bootstrapper.ts deleted file mode 100644 index 2c10811b21c48..0000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_execution_log_bootstrapper.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../../../../../rule_registry/common/assets'; -import { mappingFromFieldMap } from '../../../../../../rule_registry/common/mapping_from_field_map'; -import { IRuleDataPluginService } from '../types'; -import { ruleExecutionFieldMap } from './rule_execution_field_map'; - -/** - * @deprecated bootstrapRuleExecutionLog is kept here only as a reference. It will be superseded with EventLog implementation - */ -export const bootstrapRuleExecutionLog = async ( - ruleDataService: IRuleDataPluginService, - indexAlias: string -) => { - const indexPattern = `${indexAlias}*`; - const componentTemplateName = `${indexAlias}-mappings`; - const indexTemplateName = `${indexAlias}-template`; - - await ruleDataService.createOrUpdateComponentTemplate({ - name: componentTemplateName, - body: { - template: { - settings: { - number_of_shards: 1, - }, - mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'), - }, - }, - }); - - await ruleDataService.createOrUpdateIndexTemplate({ - name: indexTemplateName, - body: { - index_patterns: [indexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], - }, - }); - - await ruleDataService.updateIndexMappingsMatchingPattern(indexPattern); -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts index 1c19859eef432..3f2f34c17679f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/rule_registry_log_client/rule_registry_log_client.ts @@ -15,9 +15,10 @@ import { TIMESTAMP, ALERT_RULE_ID, } from '@kbn/rule-data-utils'; -import { once } from 'lodash/fp'; import moment from 'moment'; -import { RuleDataClient } from '../../../../../../rule_registry/server'; + +import { mappingFromFieldMap } from '../../../../../../rule_registry/common/mapping_from_field_map'; +import { Dataset, IRuleDataClient } from '../../../../../../rule_registry/server'; import { SERVER_APP_ID } from '../../../../../common/constants'; import { RuleExecutionStatus } from '../../../../../common/detection_engine/schemas/common/schemas'; import { invariant } from '../../../../../common/utils/invariant'; @@ -29,15 +30,9 @@ import { IRuleDataPluginService, LogStatusChangeArgs, } from '../types'; -import { - EVENTS_INDEX_PREFIX, - EVENT_SEQUENCE, - MESSAGE, - RULE_STATUS, - RULE_STATUS_SEVERITY, -} from './constants'; +import { EVENT_SEQUENCE, MESSAGE, RULE_STATUS, RULE_STATUS_SEVERITY } from './constants'; import { parseRuleExecutionLog, RuleExecutionEvent } from './parse_rule_execution_log'; -import { bootstrapRuleExecutionLog } from './rule_execution_log_bootstrapper'; +import { ruleExecutionFieldMap } from './rule_execution_field_map'; import { getLastEntryAggregation, getMetricAggregation, @@ -76,20 +71,27 @@ interface IRuleRegistryLogClient { */ export class RuleRegistryLogClient implements IRuleRegistryLogClient { private sequence = 0; - private ruleDataClient: RuleDataClient; + private ruleDataClient: IRuleDataClient; constructor(ruleDataService: IRuleDataPluginService) { - this.ruleDataClient = ruleDataService.getRuleDataClient( - SERVER_APP_ID, - EVENTS_INDEX_PREFIX, - () => this.initialize(ruleDataService, EVENTS_INDEX_PREFIX) - ); + this.ruleDataClient = ruleDataService.initializeIndex({ + feature: SERVER_APP_ID, + registrationContext: 'security', + dataset: Dataset.events, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + version: 0, + mappings: mappingFromFieldMap(ruleExecutionFieldMap, 'strict'), + }, + ], + indexTemplate: { + version: 0, + }, + }); } - private initialize = once(async (ruleDataService: IRuleDataPluginService, indexAlias: string) => { - await bootstrapRuleExecutionLog(ruleDataService, indexAlias); - }); - public async find({ ruleIds, spaceId, statuses, logsCount = 1 }: FindExecutionLogArgs) { if (ruleIds.length === 0) { return {}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts index 151b1d8b2fb13..3c28551a71deb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts @@ -11,7 +11,7 @@ import { v4 } from 'uuid'; import { Logger, SavedObject } from 'kibana/server'; import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks'; -import type { RuleDataClient } from '../../../../../../rule_registry/server'; +import type { IRuleDataClient } from '../../../../../../rule_registry/server'; import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../../alerting/server'; import { ConfigType } from '../../../../config'; import { AlertAttributes } from '../../signals/types'; @@ -89,7 +89,8 @@ export const createRuleTypeMocks = ( bulk: jest.fn(), })), isWriteEnabled: jest.fn(() => true), - } as unknown) as RuleDataClient, + indexName: '.alerts-security.alerts', + } as unknown) as IRuleDataClient, ruleDataService: ruleRegistryMocks.createRuleDataPluginService(), }, services, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts index 168502d120b8f..376a4a29ed89a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_factory.ts @@ -36,7 +36,6 @@ import { RuleExecutionLogClient } from '../rule_execution_log/rule_execution_log /* eslint-disable complexity */ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ - indexAlias, lists, logger, mergeStrategy, @@ -83,7 +82,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ id: alertId, ruleId, name, - index: indexAlias, + index: ruleDataClient.indexName, }); logger.debug(buildRuleMessage('[+] Starting Signal Rule execution')); @@ -273,7 +272,7 @@ export const createSecurityRuleTypeFactory: CreateSecurityRuleTypeFactory = ({ logger.debug(buildRuleMessage('[+] Signal Rule execution completed.')); logger.debug( buildRuleMessage( - `[+] Finished indexing ${createdSignalsCount} signals into alias ${indexAlias}` + `[+] Finished indexing ${createdSignalsCount} signals into ${ruleDataClient.indexName}` ) ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts index 50aff00191396..4a9d1b5658317 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.test.ts @@ -61,7 +61,6 @@ describe('Indicator Match Alerts', () => { const { services, dependencies, executor } = createRuleTypeMocks('threat_match', params); const indicatorMatchAlertType = createIndicatorMatchAlertType({ experimentalFeatures: allowedExperimentalValues, - indexAlias: 'alerts.security-alerts', lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', @@ -102,7 +101,6 @@ describe('Indicator Match Alerts', () => { const { services, dependencies, executor } = createRuleTypeMocks('threat_match', params); const indicatorMatchAlertType = createIndicatorMatchAlertType({ experimentalFeatures: allowedExperimentalValues, - indexAlias: 'alerts.security-alerts', lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', @@ -141,7 +139,6 @@ describe('Indicator Match Alerts', () => { const { services, dependencies, executor } = createRuleTypeMocks('threat_match', params); const indicatorMatchAlertType = createIndicatorMatchAlertType({ experimentalFeatures: allowedExperimentalValues, - indexAlias: 'alerts.security-alerts', lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts index 61342981479ae..71acc2e1cee85 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/create_indicator_match_alert_type.ts @@ -16,7 +16,6 @@ import { CreateRuleOptions } from '../types'; export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) => { const { experimentalFeatures, - indexAlias, lists, logger, mergeStrategy, @@ -25,7 +24,6 @@ export const createIndicatorMatchAlertType = (createOptions: CreateRuleOptions) ruleDataService, } = createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ - indexAlias, lists, logger, mergeStrategy, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts index a5ecc92df8f27..dfe83e32114d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.test.ts @@ -36,7 +36,6 @@ describe('Custom query alerts', () => { const { services, dependencies, executor } = createRuleTypeMocks(); const queryAlertType = createQueryAlertType({ experimentalFeatures: allowedExperimentalValues, - indexAlias: 'alerts.security-alerts', lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', @@ -84,7 +83,6 @@ describe('Custom query alerts', () => { const { services, dependencies, executor } = createRuleTypeMocks(); const queryAlertType = createQueryAlertType({ experimentalFeatures: allowedExperimentalValues, - indexAlias: 'alerts.security-alerts', lists: dependencies.lists, logger: dependencies.logger, mergeStrategy: 'allFields', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts index b66e76f598bf4..e59037f38ce56 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/create_query_alert_type.ts @@ -16,7 +16,6 @@ import { CreateRuleOptions } from '../types'; export const createQueryAlertType = (createOptions: CreateRuleOptions) => { const { experimentalFeatures, - indexAlias, lists, logger, mergeStrategy, @@ -25,7 +24,6 @@ export const createQueryAlertType = (createOptions: CreateRuleOptions) => { ruleDataService, } = createOptions; const createSecurityRuleType = createSecurityRuleTypeFactory({ - indexAlias, lists, logger, mergeStrategy, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts index f0f11f470bc6c..e781bfc50bee4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts @@ -23,7 +23,7 @@ import { TypeOfFieldMap } from '../../../../../rule_registry/common/field_map'; import { AlertTypeWithExecutor, PersistenceServices, - RuleDataClient, + IRuleDataClient, } from '../../../../../rule_registry/server'; import { BaseHit } from '../../../../common/detection_engine/types'; import { ConfigType } from '../../../config'; @@ -93,11 +93,10 @@ type SecurityAlertTypeWithExecutor< }; export type CreateSecurityRuleTypeFactory = (options: { - indexAlias: string; lists: SetupPlugins['lists']; logger: Logger; mergeStrategy: ConfigType['alertMergeStrategy']; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; ruleDataService: IRuleDataPluginService; }) => < TParams extends RuleParams & { index: string[] | undefined }, @@ -122,11 +121,10 @@ export type WrappedRACAlert = BaseHit; export interface CreateRuleOptions { experimentalFeatures: ExperimentalFeatures; - indexAlias: string; lists: SetupPlugins['lists']; logger: Logger; mergeStrategy: ConfigType['alertMergeStrategy']; - ruleDataClient: RuleDataClient; + ruleDataClient: IRuleDataClient; version: string; ruleDataService: IRuleDataPluginService; } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 06d8e35cc0102..9176c7ee70ba1 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { once } from 'lodash'; import { Observable } from 'rxjs'; import LRU from 'lru-cache'; import { estypes } from '@elastic/elasticsearch'; @@ -30,15 +29,13 @@ import { import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; import { PluginStartContract as CasesPluginStartContract } from '../../cases/server'; -import { - ECS_COMPONENT_TEMPLATE_NAME, - TECHNICAL_COMPONENT_TEMPLATE_NAME, -} from '../../rule_registry/common/assets'; +import { ECS_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; import { SecurityPluginSetup as SecuritySetup, SecurityPluginStart } from '../../security/server'; import { - RuleDataClient, + IRuleDataClient, RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract, + Dataset, } from '../../rule_registry/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; @@ -206,74 +203,46 @@ export class Plugin implements IPlugin { - if (!ruleDataService.isWriteEnabled()) { - return; - } - - // TODO: convert the aliases to FieldMaps. Requires enhancing FieldMap to support alias path. - // Split aliases by component template since we need to alias some fields in technical field mappings, - // some fields in security solution specific component template. - const aliases: Record = {}; - Object.entries(aadFieldConversion).forEach(([key, value]) => { - aliases[key] = { - type: 'alias', - path: value, - }; - }); + let ruleDataClient: IRuleDataClient | null = null; - const componentTemplateName = ruleDataService.getFullAssetName('security.alerts-mappings'); - await ruleDataService.createOrUpdateComponentTemplate({ - name: componentTemplateName, - body: { - template: { - settings: { - number_of_shards: 1, - }, - mappings: mappingFromFieldMap( - { ...alertsFieldMap, ...rulesFieldMap, ...ctiFieldMap }, - false - ), - }, - }, - }); - - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName('security.alerts-index-template'), - body: { - index_patterns: [alertsIndexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - ruleDataService.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], - }, - }); - await ruleDataService.updateIndexMappingsMatchingPattern(alertsIndexPattern); + if (isRuleRegistryEnabled) { + // NOTE: this is not used yet + // TODO: convert the aliases to FieldMaps. Requires enhancing FieldMap to support alias path. + // Split aliases by component template since we need to alias some fields in technical field mappings, + // some fields in security solution specific component template. + const aliases: Record = {}; + Object.entries(aadFieldConversion).forEach(([key, value]) => { + aliases[key] = { + type: 'alias', + path: value, + }; }); - // initialize eagerly - const initializeRuleDataTemplatesPromise = initializeRuleDataTemplates().catch((err) => { - this.logger!.error(err); + ruleDataClient = ruleDataService.initializeIndex({ + feature: SERVER_APP_ID, + registrationContext: 'security', + dataset: Dataset.alerts, + componentTemplateRefs: [ECS_COMPONENT_TEMPLATE_NAME], + componentTemplates: [ + { + name: 'mappings', + version: 0, + mappings: mappingFromFieldMap( + { ...alertsFieldMap, ...rulesFieldMap, ...ctiFieldMap }, + false + ), + }, + ], + indexTemplate: { + version: 0, + }, + secondaryAlias: config.signalsIndex, }); - const indexAlias = ruleDataService.getFullAssetName('security.alerts'); - - ruleDataClient = ruleDataService.getRuleDataClient( - SERVER_APP_ID, - indexAlias, - () => initializeRuleDataTemplatesPromise - ); - // Register rule types via rule-registry const createRuleOptions: CreateRuleOptions = { experimentalFeatures, - indexAlias, lists: plugins.lists, logger: this.logger, mergeStrategy: this.config.alertMergeStrategy, diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index c029e421ab81e..7c7a38319b1ea 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { RuleDataClient, RuleDataPluginService } from '../../../rule_registry/server'; +import { IRuleDataClient, RuleDataPluginService } from '../../../rule_registry/server'; import { SecuritySolutionPluginRouter } from '../types'; @@ -64,7 +64,7 @@ export const initRoutes = ( security: SetupPlugins['security'], ml: SetupPlugins['ml'], ruleDataService: RuleDataPluginService, - ruleDataClient: RuleDataClient | null + ruleDataClient: IRuleDataClient | null ) => { // Detection Engine Rule routes that have the REST endpoints of /api/detection_engine/rules // All REST rule creation, deletion, updating, etc...... diff --git a/x-pack/plugins/uptime/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/uptime/server/lib/alerts/test_utils/index.ts index 8bbf20f3a64ad..1927162f64772 100644 --- a/x-pack/plugins/uptime/server/lib/alerts/test_utils/index.ts +++ b/x-pack/plugins/uptime/server/lib/alerts/test_utils/index.ts @@ -9,7 +9,7 @@ import { Logger } from 'kibana/server'; import { UMServerLibs } from '../../lib'; import { UptimeCorePlugins, UptimeCoreSetup } from '../../adapters'; import type { UptimeRouter } from '../../../types'; -import type { RuleDataClient } from '../../../../../rule_registry/server'; +import type { IRuleDataClient } from '../../../../../rule_registry/server'; import { getUptimeESMockClient } from '../../requests/helper'; import { alertsMock } from '../../../../../alerting/server/mocks'; import { DynamicSettings } from '../../../../common/runtime_types'; @@ -72,7 +72,9 @@ export const createRuleTypeMocks = ( bulk: jest.fn(), }; }, - } as unknown) as RuleDataClient, + isWriteEnabled: jest.fn(() => true), + indexName: '.alerts-observability.synthetics.alerts', + } as unknown) as IRuleDataClient, }, services, scheduleActions, diff --git a/x-pack/plugins/uptime/server/plugin.ts b/x-pack/plugins/uptime/server/plugin.ts index d4bc9ca1b4d5d..3e935eab153ac 100644 --- a/x-pack/plugins/uptime/server/plugin.ts +++ b/x-pack/plugins/uptime/server/plugin.ts @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { once } from 'lodash'; + import { PluginInitializerContext, CoreStart, @@ -18,7 +18,7 @@ import { initServerWithKibana } from './kibana.index'; import { KibanaTelemetryAdapter, UptimeCorePlugins } from './lib/adapters'; import { umDynamicSettings } from './lib/saved_objects'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; -import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; +import { Dataset } from '../../rule_registry/server'; export type UptimeRuleRegistry = ReturnType['ruleRegistry']; @@ -35,51 +35,23 @@ export class Plugin implements PluginType { this.logger = this.initContext.logger.get(); const { ruleDataService } = plugins.ruleRegistry; - const ready = once(async () => { - const componentTemplateName = ruleDataService.getFullAssetName('synthetics-mappings'); - const alertsIndexPattern = ruleDataService.getFullAssetName('observability.synthetics*'); - - if (!ruleDataService.isWriteEnabled()) { - return; - } - - await ruleDataService.createOrUpdateComponentTemplate({ - name: componentTemplateName, - body: { - template: { - settings: { - number_of_shards: 1, - }, - mappings: mappingFromFieldMap(uptimeRuleFieldMap, 'strict'), - }, - }, - }); - - await ruleDataService.createOrUpdateIndexTemplate({ - name: ruleDataService.getFullAssetName('synthetics-index-template'), - body: { - index_patterns: [alertsIndexPattern], - composed_of: [ - ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), - componentTemplateName, - ], + const ruleDataClient = ruleDataService.initializeIndex({ + feature: 'synthetics', + registrationContext: 'observability.synthetics', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + version: 0, + mappings: mappingFromFieldMap(uptimeRuleFieldMap, 'strict'), }, - }); - - await ruleDataService.updateIndexMappingsMatchingPattern(alertsIndexPattern); - }); - - // initialize eagerly - const initializeRuleDataTemplatesPromise = ready().catch((err) => { - this.logger!.error(err); + ], + indexTemplate: { + version: 0, + }, }); - const ruleDataClient = ruleDataService.getRuleDataClient( - 'synthetics', - ruleDataService.getFullAssetName('observability.synthetics'), - () => initializeRuleDataTemplatesPromise - ); - initServerWithKibana( { router: core.http.createRouter() }, plugins, diff --git a/x-pack/plugins/uptime/server/uptime_server.ts b/x-pack/plugins/uptime/server/uptime_server.ts index f52b4a8063351..ded76027a3c3a 100644 --- a/x-pack/plugins/uptime/server/uptime_server.ts +++ b/x-pack/plugins/uptime/server/uptime_server.ts @@ -6,7 +6,7 @@ */ import { Logger } from 'kibana/server'; -import { createLifecycleRuleTypeFactory, RuleDataClient } from '../../rule_registry/server'; +import { createLifecycleRuleTypeFactory, IRuleDataClient } from '../../rule_registry/server'; import { UMServerLibs } from './lib/lib'; import { createRouteWithAuth, restApiRoutes, uptimeRouteWrapper } from './rest_api'; import { UptimeCoreSetup, UptimeCorePlugins } from './lib/adapters'; @@ -20,7 +20,7 @@ export const initUptimeServer = ( server: UptimeCoreSetup, libs: UMServerLibs, plugins: UptimeCorePlugins, - ruleDataClient: RuleDataClient, + ruleDataClient: IRuleDataClient, logger: Logger ) => { restApiRoutes.forEach((route) => diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts index 857101ebcc18c..0d318ea5bc6c7 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -42,7 +42,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const BULK_INDEX_DELAY = 1000; const INDEXING_DELAY = 5000; - const ALERTS_INDEX_TARGET = '.kibana-alerts-*-apm*'; + const ALERTS_INDEX_TARGET = '.kibana-alerts-observability.apm.alerts*'; const APM_TRANSACTION_INDEX_NAME = 'apm-8.0.0-transaction'; const createTransactionEvent = (override: Record) => {