[RAC][Rule Registry] Improve RuleDataService API and index bootstrapp…

…ing implementation (#108115) (#108638) **Addresses:** #106421, #106428, #102089, #106433 ## Summary This PR focuses on consolidation of indexing implementations in `rule_registry` (#101016). It addresses some of the sub-tasks of the parent ticket. - [x] Encapsulate index bootstrapping logic in a new improved API exposed by `RuleDataService`. - [x] Enforce allowed values for the `datasetSuffix` on the API level. - [x] Migrate plugins using the existing `RuleDataService` API to the improved one. - [x] Make sure index names comply with design architecture. - #102089 - [x] Improve the API of `RuleDataClient`. - [x] Enhance index bootstrapping: support custom ILM policy per index (`{registrationContext}.{datasetSuffix}`). - [x] Enhance index bootstrapping: create index template per namespace and support rollovers properly - based on #107700 - [x] Enhance index bootstrapping: support secondary aliases - based on #107700 - [x] Remove `EventLogService` implementation - #106433 This will be addressed in follow-up PRs: - [ ] Enhance index bootstrapping: implement suggestions for backwards compatibility (naming scheme for alias and backing indices; versioning). - [ ] Enhance index bootstrapping: implement upgrades of existing index templates. - [ ] Make index bootstrapping logic more robust. This _is partially addressed_ in this PR, but more improvements are needed. - [ ] Change the way index prefix works. - [ ] Add support for optional TS schema (static typing). - [ ] Update `README` in `rule_registry`. ### Checklist - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios Co-authored-by: Georgii Gorbachev <georgii.gorbachev@elastic.co>
elastic · Aug 15, 2021 · 51244c8 · 51244c8
1 parent 0f406c0
commit 51244c8
Show file tree

Hide file tree

Showing 88 changed files with 1,266 additions and 2,596 deletions.
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts
@@ -8,7 +8,7 @@
 import { Observable } from 'rxjs';
 import { Logger } from 'kibana/server';
 import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../alerting/server';
-import { RuleDataClient } from '../../../../rule_registry/server';
+import { IRuleDataClient } from '../../../../rule_registry/server';
 import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type';
 import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type';
 import { registerErrorCountAlertType } from './register_error_count_alert_type';
@@ -17,7 +17,7 @@ import { MlPluginSetup } from '../../../../ml/server';
 import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type';
 
 export interface RegisterRuleDependencies {
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
   ml?: MlPluginSetup;
   alerting: AlertingPluginSetupContract;
   config$: Observable<APMConfig>;

diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts
@@ -8,7 +8,7 @@
 import { Logger } from 'kibana/server';
 import { of } from 'rxjs';
 import { elasticsearchServiceMock } from 'src/core/server/mocks';
-import type { RuleDataClient } from '../../../../../rule_registry/server';
+import type { IRuleDataClient } from '../../../../../rule_registry/server';
 import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server';
 import { APMConfig, APM_SERVER_FEATURE_ID } from '../../..';
 
@@ -63,7 +63,8 @@ export const createRuleTypeMocks = () => {
           };
         },
         isWriteEnabled: jest.fn(() => true),
-      } as unknown) as RuleDataClient,
+        indexName: '.alerts-observability.apm.alerts',
+      } as unknown) as IRuleDataClient,
     },
     services,
     scheduleActions,

diff --git a/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts b/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts
@@ -8,7 +8,7 @@
 import type { EVENT_KIND as EVENT_KIND_TYPED } from '@kbn/rule-data-utils';
 // @ts-expect-error
 import { EVENT_KIND as EVENT_KIND_NON_TYPED } from '@kbn/rule-data-utils/target_node/technical_field_names';
-import { RuleDataClient } from '../../../../rule_registry/server';
+import { IRuleDataClient } from '../../../../rule_registry/server';
 import {
   SERVICE_NAME,
   TRANSACTION_TYPE,
@@ -26,7 +26,7 @@ export async function getServiceAlerts({
   environment,
   transactionType,
 }: {
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
   start: number;
   end: number;
   serviceName: string;

diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts
@@ -15,10 +15,10 @@ import {
   Plugin,
   PluginInitializerContext,
 } from 'src/core/server';
-import { isEmpty, mapValues, once } from 'lodash';
+import { isEmpty, mapValues } from 'lodash';
 import { SavedObjectsClient } from '../../../../src/core/server';
-import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets';
 import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map';
+import { Dataset } from '../../rule_registry/server';
 import { APMConfig, APMXPackConfig, APM_SERVER_FEATURE_ID } from '.';
 import { mergeConfigs } from './index';
 import { UI_SETTINGS } from '../../../../src/plugins/data/common';
@@ -106,79 +106,43 @@ export class APMPlugin
 
     registerFeaturesUsage({ licensingPlugin: plugins.licensing });
 
-    const { ruleDataService } = plugins.ruleRegistry;
     const getCoreStart = () =>
       core.getStartServices().then(([coreStart]) => coreStart);
 
-    const alertsIndexPattern = ruleDataService.getFullAssetName(
-      'observability-apm*'
-    );
-
-    const initializeRuleDataTemplates = once(async () => {
-      const componentTemplateName = ruleDataService.getFullAssetName(
-        'apm-mappings'
-      );
-
-      if (!ruleDataService.isWriteEnabled()) {
-        return;
-      }
-
-      await ruleDataService.createOrUpdateComponentTemplate({
-        name: componentTemplateName,
-        body: {
-          template: {
-            settings: {
-              number_of_shards: 1,
-            },
-            mappings: mappingFromFieldMap(
-              {
-                [SERVICE_NAME]: {
-                  type: 'keyword',
-                },
-                [SERVICE_ENVIRONMENT]: {
-                  type: 'keyword',
-                },
-                [TRANSACTION_TYPE]: {
-                  type: 'keyword',
-                },
-                [PROCESSOR_EVENT]: {
-                  type: 'keyword',
-                },
+    const { ruleDataService } = plugins.ruleRegistry;
+    const ruleDataClient = ruleDataService.initializeIndex({
+      feature: APM_SERVER_FEATURE_ID,
+      registrationContext: 'observability.apm',
+      dataset: Dataset.alerts,
+      componentTemplateRefs: [],
+      componentTemplates: [
+        {
+          name: 'mappings',
+          version: 0,
+          mappings: mappingFromFieldMap(
+            {
+              [SERVICE_NAME]: {
+                type: 'keyword',
               },
-              'strict'
-            ),
-          },
-        },
-      });
-
-      await ruleDataService.createOrUpdateIndexTemplate({
-        name: ruleDataService.getFullAssetName('apm-index-template'),
-        body: {
-          index_patterns: [alertsIndexPattern],
-          composed_of: [
-            ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
-            componentTemplateName,
-          ],
+              [SERVICE_ENVIRONMENT]: {
+                type: 'keyword',
+              },
+              [TRANSACTION_TYPE]: {
+                type: 'keyword',
+              },
+              [PROCESSOR_EVENT]: {
+                type: 'keyword',
+              },
+            },
+            'strict'
+          ),
         },
-      });
-      await ruleDataService.updateIndexMappingsMatchingPattern(
-        alertsIndexPattern
-      );
+      ],
+      indexTemplate: {
+        version: 0,
+      },
     });
 
-    // initialize eagerly
-    const initializeRuleDataTemplatesPromise = initializeRuleDataTemplates().catch(
-      (err) => {
-        this.logger!.error(err);
-      }
-    );
-
-    const ruleDataClient = ruleDataService.getRuleDataClient(
-      APM_SERVER_FEATURE_ID,
-      ruleDataService.getFullAssetName('observability-apm'),
-      () => initializeRuleDataTemplatesPromise
-    );
-
     const resourcePlugins = mapValues(plugins, (value, key) => {
       return {
         setup: value,

diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts
@@ -12,7 +12,7 @@ import {
   KibanaRequest,
   CoreStart,
 } from 'src/core/server';
-import { RuleDataClient } from '../../../rule_registry/server';
+import { IRuleDataClient } from '../../../rule_registry/server';
 import { AlertingApiRequestHandlerContext } from '../../../alerting/server';
 import type { RacApiRequestHandlerContext } from '../../../rule_registry/server';
 import { LicensingApiRequestHandlerContext } from '../../../licensing/server';
@@ -72,6 +72,6 @@ export interface APMRouteHandlerResources {
       start: () => Promise<Required<APMPluginDependencies>[key]['start']>;
     };
   };
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
   telemetryUsageCounter?: TelemetryUsageCounter;
 }
diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts
@@ -5,10 +5,8 @@
  * 2.0.
  */
 
-import { once } from 'lodash';
 import { CoreSetup, Logger } from 'src/core/server';
-import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../../../rule_registry/common/assets';
-import { RuleRegistryPluginSetupContract } from '../../../../rule_registry/server';
+import { Dataset, RuleRegistryPluginSetupContract } from '../../../../rule_registry/server';
 import type { InfraFeatureId } from '../../../common/constants';
 import { RuleRegistrationContext, RulesServiceStartDeps } from './types';
 
@@ -25,51 +23,20 @@ export const createRuleDataClient = ({
   logger: Logger;
   ruleDataService: RuleRegistryPluginSetupContract['ruleDataService'];
 }) => {
-  const initializeRuleDataTemplates = once(async () => {
-    const componentTemplateName = ruleDataService.getFullAssetName(
-      `${registrationContext}-mappings`
-    );
-
-    const indexNamePattern = ruleDataService.getFullAssetName(`${registrationContext}*`);
-
-    if (!ruleDataService.isWriteEnabled()) {
-      return;
-    }
-
-    await ruleDataService.createOrUpdateComponentTemplate({
-      name: componentTemplateName,
-      body: {
-        template: {
-          settings: {
-            number_of_shards: 1,
-          },
-          mappings: {},
-        },
-      },
-    });
-
-    await ruleDataService.createOrUpdateIndexTemplate({
-      name: ruleDataService.getFullAssetName(registrationContext),
-      body: {
-        index_patterns: [indexNamePattern],
-        composed_of: [
-          ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME),
-          componentTemplateName,
-        ],
+  return ruleDataService.initializeIndex({
+    feature: ownerFeatureId,
+    registrationContext,
+    dataset: Dataset.alerts,
+    componentTemplateRefs: [],
+    componentTemplates: [
+      {
+        name: 'mappings',
+        version: 0,
+        mappings: {},
       },
-    });
-
-    await ruleDataService.updateIndexMappingsMatchingPattern(indexNamePattern);
-  });
-
-  // initialize eagerly
-  const initializeRuleDataTemplatesPromise = initializeRuleDataTemplates().catch((err) => {
-    logger.error(err);
+    ],
+    indexTemplate: {
+      version: 0,
+    },
   });
-
-  return ruleDataService.getRuleDataClient(
-    ownerFeatureId,
-    ruleDataService.getFullAssetName(registrationContext),
-    () => initializeRuleDataTemplatesPromise
-  );
 };
diff --git a/x-pack/plugins/infra/server/services/rules/types.ts b/x-pack/plugins/infra/server/services/rules/types.ts
@@ -8,7 +8,7 @@
 import { PluginSetupContract as AlertingPluginSetup } from '../../../../alerting/server';
 import {
   createLifecycleExecutor,
-  RuleDataClient,
+  IRuleDataClient,
   RuleRegistryPluginSetupContract,
 } from '../../../../rule_registry/server';
 
@@ -24,7 +24,7 @@ export interface RulesServiceStartDeps {}
 
 export interface RulesServiceSetup {
   createLifecycleRuleExecutor: LifecycleRuleExecutorCreator;
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface

diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts
@@ -18,7 +18,7 @@ import {
   ScopedAnnotationsClientFactory,
   AnnotationsAPI,
 } from './lib/annotations/bootstrap_annotations';
-import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server';
+import { Dataset, RuleRegistryPluginSetupContract } from '../../rule_registry/server';
 import { PluginSetupContract as FeaturesSetup } from '../../features/server';
 import { uiSettings } from './ui_settings';
 import { registerRoutes } from './routes/register_routes';
@@ -100,11 +100,17 @@ export class ObservabilityPlugin implements Plugin<ObservabilityPluginSetup> {
 
     const start = () => core.getStartServices().then(([coreStart]) => coreStart);
 
-    const ruleDataClient = plugins.ruleRegistry.ruleDataService.getRuleDataClient(
-      'observability',
-      plugins.ruleRegistry.ruleDataService.getFullAssetName(),
-      () => Promise.resolve()
-    );
+    const { ruleDataService } = plugins.ruleRegistry;
+    const ruleDataClient = ruleDataService.initializeIndex({
+      feature: 'observability',
+      registrationContext: 'observability',
+      dataset: Dataset.alerts,
+      componentTemplateRefs: [],
+      componentTemplates: [],
+      indexTemplate: {
+        version: 0,
+      },
+    });
 
     registerRoutes({
       core: {

diff --git a/x-pack/plugins/observability/server/routes/register_routes.ts b/x-pack/plugins/observability/server/routes/register_routes.ts
@@ -13,7 +13,7 @@ import {
 import { CoreSetup, CoreStart, Logger, RouteRegistrar } from 'kibana/server';
 import Boom from '@hapi/boom';
 import { RequestAbortedError } from '@elastic/elasticsearch/lib/errors';
-import { RuleDataClient } from '../../../rule_registry/server';
+import { IRuleDataClient } from '../../../rule_registry/server';
 import { ObservabilityRequestHandlerContext } from '../types';
 import { AbstractObservabilityServerRouteRepository } from './types';
 
@@ -29,7 +29,7 @@ export function registerRoutes({
   };
   repository: AbstractObservabilityServerRouteRepository;
   logger: Logger;
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
 }) {
   const routes = repository.getRoutes();
 

diff --git a/x-pack/plugins/observability/server/routes/types.ts b/x-pack/plugins/observability/server/routes/types.ts
@@ -12,7 +12,7 @@ import type {
   ServerRouteRepository,
 } from '@kbn/server-route-repository';
 import { CoreSetup, CoreStart, KibanaRequest, Logger } from 'kibana/server';
-import { RuleDataClient } from '../../../rule_registry/server';
+import { IRuleDataClient } from '../../../rule_registry/server';
 
 import { ObservabilityServerRouteRepository } from './get_global_observability_server_route_repository';
 import { ObservabilityRequestHandlerContext } from '../types';
@@ -24,7 +24,7 @@ export interface ObservabilityRouteHandlerResources {
     start: () => Promise<CoreStart>;
     setup: CoreSetup;
   };
-  ruleDataClient: RuleDataClient;
+  ruleDataClient: IRuleDataClient;
   request: KibanaRequest;
   context: ObservabilityRequestHandlerContext;
   logger: Logger;

diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts
@@ -7,10 +7,6 @@
 
 import { estypes } from '@elastic/elasticsearch';
 
-export type PutIndexTemplateRequest = estypes.IndicesPutIndexTemplateRequest & {
-  body?: { composed_of?: string[] };
-};
-
 export interface ClusterPutComponentTemplateBody {
   template: {
     settings: {

diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json
@@ -9,7 +9,6 @@
   "requiredPlugins": [
     "alerting",
     "data",
-    "spaces",
     "triggersActionsUi"
   ],
   "optionalPlugins": ["security"],

diff --git a/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts b/x-pack/plugins/rule_registry/server/event_log/elasticsearch/index.ts