[RAC][Rule Registry] Implement versioning and backing indices (#109276)…

… (#110126)

**Ticket:** #109293

🚨 **This PR is critical for Observability 7.15** 🚨

## Summary

This PR fixes the indexing implementation in `rule_registry`. It implements the suggestions for backwards compatibility described in the ticket:

- changes the naming scheme and introduces the concept of "backing indices", so that names of the concrete ("backing") indices != names of their aliases
- adds versioning based on the current Kibana version

TODO:

- [x] Change index naming (implement the concept of backing indices)
- [x] Include Kibana version into the index template metadata
- [x] Include Kibana version into the document fields
- [x] Remove `version` from `IndexOptions` (parameters provided by solutions/plugins when initializing alerts-as-data indices)
- [x] Fix CI

### Checklist

- [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials
- [ ] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

Co-authored-by: Georgii Gorbachev <georgii.gorbachev@elastic.co>

x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts

@@ -8,7 +8,8 @@
 import { Logger } from 'kibana/server';
 import { of } from 'rxjs';
 import { elasticsearchServiceMock } from 'src/core/server/mocks';
-import type { IRuleDataClient } from '../../../../../rule_registry/server';
+import { IRuleDataClient } from '../../../../../rule_registry/server';
+import { ruleRegistryMocks } from '../../../../../rule_registry/server/mocks';
 import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server';
 import { APMConfig, APM_SERVER_FEATURE_ID } from '../../..';
 
@@ -51,20 +52,9 @@ export const createRuleTypeMocks = () => {
       alerting,
       config$: mockedConfig$,
       logger: loggerMock,
-      ruleDataClient: ({
-        getReader: () => {
-          return {
-            search: jest.fn(),
-          };
-        },
-        getWriter: () => {
-          return {
-            bulk: jest.fn(),
-          };
-        },
-        isWriteEnabled: jest.fn(() => true),
-        indexName: '.alerts-observability.apm.alerts',
-      } as unknown) as IRuleDataClient,
+      ruleDataClient: ruleRegistryMocks.createRuleDataClient(
+        '.alerts-observability.apm.alerts'
+      ) as IRuleDataClient,
     },
     services,
     scheduleActions,

x-pack/plugins/apm/server/plugin.ts

@@ -122,7 +122,6 @@ export class APMPlugin
       componentTemplates: [
         {
           name: 'mappings',
-          version: 0,
           mappings: mappingFromFieldMap(
             {
               [SERVICE_NAME]: {
@@ -142,9 +141,6 @@ export class APMPlugin
           ),
         },
       ],
-      indexTemplate: {
-        version: 0,
-      },
     });
 
     const resourcePlugins = mapValues(plugins, (value, key) => {

x-pack/plugins/infra/server/services/rules/rule_data_client.ts

@@ -31,12 +31,8 @@ export const createRuleDataClient = ({
     componentTemplates: [
       {
         name: 'mappings',
-        version: 0,
         mappings: {},
       },
     ],
-    indexTemplate: {
-      version: 0,
-    },
   });
 };

x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts

@@ -30,7 +30,7 @@ export const technicalRuleFieldMap = {
   [Fields.ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 },
   [Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 },
   [Fields.VERSION]: {
-    type: 'keyword',
+    type: 'version',
     array: false,
     required: false,
   },

x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts

@@ -45,6 +45,7 @@ const BooleanFromString = new t.Type(
 
 const esFieldTypeMap = {
   keyword: t.string,
+  version: t.string,
   text: t.string,
   date: t.string,
   boolean: t.union([t.number, BooleanFromString]),

x-pack/plugins/rule_registry/server/config.ts

@@ -24,3 +24,4 @@ export const config = {
 export type RuleRegistryPluginConfig = TypeOf<typeof config.schema>;
 
 export const INDEX_PREFIX = '.alerts' as const;
+export const INDEX_PREFIX_FOR_BACKING_INDICES = '.internal.alerts' as const;

x-pack/plugins/rule_registry/server/mocks.ts

@@ -6,11 +6,13 @@
  */
 
 import { alertsClientMock } from './alert_data_client/alerts_client.mock';
+import { createRuleDataClientMock } from './rule_data_client/rule_data_client.mock';
 import { ruleDataPluginServiceMock } from './rule_data_plugin_service/rule_data_plugin_service.mock';
 import { createLifecycleAlertServicesMock } from './utils/lifecycle_alert_services_mock';
 
 export const ruleRegistryMocks = {
   createLifecycleAlertServices: createLifecycleAlertServicesMock,
   createRuleDataPluginService: ruleDataPluginServiceMock.create,
+  createRuleDataClient: createRuleDataClientMock,
   createAlertsClientMock: alertsClientMock,
 };

x-pack/plugins/rule_registry/server/plugin.ts

@@ -19,7 +19,7 @@ import {
 import { PluginStartContract as AlertingStart } from '../../alerting/server';
 import { SecurityPluginSetup } from '../../security/server';
 
-import { INDEX_PREFIX, RuleRegistryPluginConfig } from './config';
+import { RuleRegistryPluginConfig } from './config';
 import { RuleDataPluginService } from './rule_data_plugin_service';
 import { AlertsClientFactory } from './alert_data_client/alerts_client_factory';
 import { AlertsClient } from './alert_data_client/alerts_client';
@@ -54,6 +54,7 @@ export class RuleRegistryPlugin
   private readonly config: RuleRegistryPluginConfig;
   private readonly legacyConfig: SharedGlobalConfig;
   private readonly logger: Logger;
+  private readonly kibanaVersion: string;
   private readonly alertsClientFactory: AlertsClientFactory;
   private ruleDataService: RuleDataPluginService | null;
   private security: SecurityPluginSetup | undefined;
@@ -63,6 +64,7 @@ export class RuleRegistryPlugin
     // TODO: Can be removed in 8.0.0. Exists to work around multi-tenancy users.
     this.legacyConfig = initContext.config.legacy.get();
     this.logger = initContext.logger.get();
+    this.kibanaVersion = initContext.env.packageInfo.version;
     this.ruleDataService = null;
     this.alertsClientFactory = new AlertsClientFactory();
   }
@@ -71,7 +73,7 @@ export class RuleRegistryPlugin
     core: CoreSetup<RuleRegistryPluginStartDependencies, RuleRegistryPluginStartContract>,
     plugins: RuleRegistryPluginSetupDependencies
   ): RuleRegistryPluginSetupContract {
-    const { logger } = this;
+    const { logger, kibanaVersion } = this;
 
     const startDependencies = core.getStartServices().then(([coreStart, pluginStart]) => {
       return {
@@ -99,8 +101,8 @@ export class RuleRegistryPlugin
 
     this.ruleDataService = new RuleDataPluginService({
       logger,
+      kibanaVersion,
       isWriteEnabled: isWriteEnabled(this.config, this.legacyConfig),
-      index: INDEX_PREFIX,
       getClusterClient: async () => {
         const deps = await startDependencies;
         return deps.core.elasticsearch.client.asInternalUser;

x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.mock.ts

@@ -18,23 +18,25 @@ type RuleDataClientMock = jest.Mocked<Omit<IRuleDataClient, 'getWriter' | 'getRe
   getWriter: (...args: Parameters<IRuleDataClient['getWriter']>) => MockInstances<IRuleDataWriter>;
 };
 
-export function createRuleDataClientMock(): RuleDataClientMock {
+export const createRuleDataClientMock = (
+  indexName: string = '.alerts-security.alerts'
+): RuleDataClientMock => {
   const bulk = jest.fn();
   const search = jest.fn();
   const getDynamicIndexPattern = jest.fn();
 
   return {
-    indexName: '.alerts-security.alerts',
-
+    indexName,
+    kibanaVersion: '7.16.0',
     isWriteEnabled: jest.fn(() => true),
 
     getReader: jest.fn((_options?: { namespace?: string }) => ({
-      getDynamicIndexPattern,
       search,
+      getDynamicIndexPattern,
     })),
 
     getWriter: jest.fn(() => ({
       bulk,
     })),
   };
-}
+};

x-pack/plugins/rule_registry/server/rule_data_client/rule_data_client.ts

@@ -33,6 +33,10 @@ export class RuleDataClient implements IRuleDataClient {
     return this.options.indexInfo.baseName;
   }
 
+  public get kibanaVersion(): string {
+    return this.options.indexInfo.kibanaVersion;
+  }
+
   public isWriteEnabled(): boolean {
     return this.options.isWriteEnabled;
   }

x-pack/plugins/rule_registry/server/rule_data_client/types.ts

@@ -14,6 +14,7 @@ import { TechnicalRuleDataFieldName } from '../../common/technical_rule_data_fie
 
 export interface IRuleDataClient {
   indexName: string;
+  kibanaVersion: string;
   isWriteEnabled(): boolean;
   getReader(options?: { namespace?: string }): IRuleDataReader;
   getWriter(options?: { namespace?: string }): IRuleDataWriter;

x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_info.ts

@@ -5,22 +5,13 @@
  * 2.0.
  */
 
+import { INDEX_PREFIX, INDEX_PREFIX_FOR_BACKING_INDICES } from '../config';
 import { IndexOptions } from './index_options';
 import { joinWithDash } from './utils';
 
 interface ConstructorOptions {
-  /**
-   * Prepends a relative resource name (defined in the code) with
-   * a full resource prefix, which starts with '.alerts' and can
-   * optionally include a user-defined part in it.
-   * @example 'security.alerts' => '.alerts-security.alerts'
-   */
-  getResourceName(relativeName: string): string;
-
-  /**
-   * Options provided by the plugin/solution defining the index.
-   */
   indexOptions: IndexOptions;
+  kibanaVersion: string;
 }
 
 /**
@@ -31,12 +22,17 @@ interface ConstructorOptions {
  */
 export class IndexInfo {
   constructor(options: ConstructorOptions) {
-    const { getResourceName, indexOptions } = options;
+    const { indexOptions, kibanaVersion } = options;
     const { registrationContext, dataset } = indexOptions;
 
     this.indexOptions = indexOptions;
-    this.baseName = getResourceName(`${registrationContext}.${dataset}`);
+    this.kibanaVersion = kibanaVersion;
+    this.baseName = joinWithDash(INDEX_PREFIX, `${registrationContext}.${dataset}`);
     this.basePattern = joinWithDash(this.baseName, '*');
+    this.baseNameForBackingIndices = joinWithDash(
+      INDEX_PREFIX_FOR_BACKING_INDICES,
+      `${registrationContext}.${dataset}`
+    );
   }
 
   /**
@@ -45,7 +41,13 @@ export class IndexInfo {
   public readonly indexOptions: IndexOptions;
 
   /**
-   * Base index name, prefixed with the full resource prefix.
+   * Current version of Kibana. We version our index resources and documents based on it.
+   * @example '7.16.0'
+   */
+  public readonly kibanaVersion: string;
+
+  /**
+   * Base index name, prefixed with the resource prefix.
    * @example '.alerts-security.alerts'
    */
   public readonly baseName: string;
@@ -56,6 +58,12 @@ export class IndexInfo {
    */
   public readonly basePattern: string;
 
+  /**
+   * Base name for internal backing indices, prefixed with a special prefix.
+   * @example '.internal.alerts-security.alerts'
+   */
+  private readonly baseNameForBackingIndices: string;
+
   /**
    * Primary index alias. Includes a namespace.
    * Used as a write target when writing documents to the index.
@@ -65,14 +73,6 @@ export class IndexInfo {
     return joinWithDash(this.baseName, namespace);
   }
 
-  /**
-   * Index pattern based on the primary alias.
-   * @example '.alerts-security.alerts-default-*'
-   */
-  public getPrimaryAliasPattern(namespace: string): string {
-    return joinWithDash(this.baseName, namespace, '*');
-  }
-
   /**
    * Optional secondary alias that can be applied to concrete indices in
    * addition to the primary one.
@@ -83,6 +83,26 @@ export class IndexInfo {
     return secondaryAlias ? joinWithDash(secondaryAlias, namespace) : null;
   }
 
+  /**
+   * Name of the initial concrete index, with the namespace and the ILM suffix.
+   * @example '.internal.alerts-security.alerts-default-000001'
+   */
+  public getConcreteIndexInitialName(namespace: string): string {
+    return joinWithDash(this.baseNameForBackingIndices, namespace, '000001');
+  }
+
+  /**
+   * Index pattern for internal backing indices. Used in the index bootstrapping logic.
+   * Can include or exclude the namespace.
+   *
+   * WARNING: Must not be used for reading documents! If you use it, you should know what you're doing.
+   *
+   * @example '.internal.alerts-security.alerts-default-*', '.internal.alerts-security.alerts-*'
+   */
+  public getPatternForBackingIndices(namespace?: string): string {
+    return joinWithDash(this.baseNameForBackingIndices, namespace, '*');
+  }
+
   /**
    * Index pattern that should be used when reading documents from the index.
    * Can include or exclude the namespace.
@@ -100,14 +120,6 @@ export class IndexInfo {
     return `${joinWithDash(this.baseName, namespace)}*`;
   }
 
-  /**
-   * Name of the initial concrete index, with the namespace and the ILM suffix.
-   * @example '.alerts-security.alerts-default-000001'
-   */
-  public getConcreteIndexInitialName(namespace: string): string {
-    return joinWithDash(this.baseName, namespace, '000001');
-  }
-
   /**
    * Name of the custom ILM policy (if it's provided by the plugin/solution).
    * Specific to the index. Shared between all namespaces of the index.

x-pack/plugins/rule_registry/server/rule_data_plugin_service/index_options.ts

@@ -75,7 +75,7 @@ export interface IndexOptions {
   /**
    * Additional properties for the namespaced index template.
    */
-  indexTemplate: IndexTemplateOptions;
+  indexTemplate?: IndexTemplateOptions;
 
   /**
    * Optional custom ILM policy for the index.
@@ -120,7 +120,6 @@ export type Meta = estypes.Metadata;
  */
 export interface ComponentTemplateOptions {
   name: string;
-  version: Version; // TODO: encapsulate versioning (base on Kibana version)
   mappings?: Mappings;
   settings?: Settings;
   _meta?: Meta;
@@ -140,7 +139,6 @@ export interface ComponentTemplateOptions {
  * https://www.elastic.co/guide/en/elasticsearch/reference/current/index-templates.html
  */
 export interface IndexTemplateOptions {
-  version: Version; // TODO: encapsulate versioning (base on Kibana version)
   _meta?: Meta;
 }