[Reporting] ILM policy for managing reporting indices (#100130)

* wip; added logic for creating ILM policy at start up

* added log when ilm policy is not created

* added test for start function

* updated ilm policy to not delete data

* actually update jest snapshots and remove unused import

* updated the ilm policy, removed the min_age for the hot phase

* update jest snapshot

* removed TODO comment

* debug log -> info log

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { PutLifecycleRequest } from '@elastic/elasticsearch/api/types';
+
+export const reportingIlmPolicy: PutLifecycleRequest['body'] = {
+  policy: {
+    phases: {
+      hot: {
+        actions: {},
+      },
+    },
+  },
+};

x-pack/plugins/reporting/server/lib/store/store.test.ts

@@ -7,6 +7,7 @@
 
 import type { DeeplyMockedKeys } from '@kbn/utility-types/jest';
 import { ElasticsearchClient } from 'src/core/server';
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
 import { ReportingCore } from '../../';
 import {
   createMockConfigSchema,
@@ -16,6 +17,8 @@ import {
 import { Report, ReportDocument } from './report';
 import { ReportingStore } from './store';
 
+const { createApiResponse } = elasticsearchServiceMock;
+
 describe('ReportingStore', () => {
   const mockLogger = createMockLevelLogger();
   let mockCore: ReportingCore;
@@ -403,4 +406,40 @@ describe('ReportingStore', () => {
       ]
     `);
   });
+
+  describe('start', () => {
+    it('creates an ILM policy for managing reporting indices if there is not already one', async () => {
+      mockEsClient.ilm.getLifecycle.mockRejectedValueOnce(createApiResponse({ statusCode: 404 }));
+      mockEsClient.ilm.putLifecycle.mockResolvedValueOnce(createApiResponse());
+
+      const store = new ReportingStore(mockCore, mockLogger);
+      await store.start();
+
+      expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' });
+      expect(mockEsClient.ilm.putLifecycle.mock.calls[0][0]).toMatchInlineSnapshot(`
+        Object {
+          "body": Object {
+            "policy": Object {
+              "phases": Object {
+                "hot": Object {
+                  "actions": Object {},
+                },
+              },
+            },
+          },
+          "policy": "kibana-reporting",
+        }
+      `);
+    });
+
+    it('does not create an ILM policy for managing reporting indices if one already exists', async () => {
+      mockEsClient.ilm.getLifecycle.mockResolvedValueOnce(createApiResponse());
+
+      const store = new ReportingStore(mockCore, mockLogger);
+      await store.start();
+
+      expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' });
+      expect(mockEsClient.ilm.putLifecycle).not.toHaveBeenCalled();
+    });
+  });
 });

x-pack/plugins/reporting/server/lib/store/store.ts

@@ -14,6 +14,7 @@ import { ReportTaskParams } from '../tasks';
 import { indexTimestamp } from './index_timestamp';
 import { mapping } from './mapping';
 import { Report, ReportDocument, ReportSource } from './report';
+import { reportingIlmPolicy } from './report_ilm_policy';
 
 /*
  * When searching for long-pending reports, we get a subset of fields
@@ -71,19 +72,22 @@ export class ReportingStore {
       return exists;
     }
 
-    const indexSettings = {
-      number_of_shards: 1,
-      auto_expand_replicas: '0-1',
-    };
-    const body = {
-      settings: indexSettings,
-      mappings: {
-        properties: mapping,
-      },
-    };
-
     try {
-      await client.indices.create({ index: indexName, body });
+      await client.indices.create({
+        index: indexName,
+        body: {
+          settings: {
+            number_of_shards: 1,
+            auto_expand_replicas: '0-1',
+            lifecycle: {
+              name: this.ilmPolicyName,
+            },
+          },
+          mappings: {
+            properties: mapping,
+          },
+        },
+      });
 
       return true;
     } catch (error) {
@@ -130,6 +134,44 @@ export class ReportingStore {
     return client.indices.refresh({ index });
   }
 
+  private readonly ilmPolicyName = 'kibana-reporting';
+
+  private async doesIlmPolicyExist(): Promise<boolean> {
+    const client = await this.getClient();
+    try {
+      await client.ilm.getLifecycle({ policy: this.ilmPolicyName });
+      return true;
+    } catch (e) {
+      if (e.statusCode === 404) {
+        return false;
+      }
+      throw e;
+    }
+  }
+
+  /**
+   * Function to be called during plugin start phase. This ensures the environment is correctly
+   * configured for storage of reports.
+   */
+  public async start() {
+    const client = await this.getClient();
+    try {
+      if (await this.doesIlmPolicyExist()) {
+        this.logger.debug(`Found ILM policy ${this.ilmPolicyName}; skipping creation.`);
+        return;
+      }
+      this.logger.info(`Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}`);
+      await client.ilm.putLifecycle({
+        policy: this.ilmPolicyName,
+        body: reportingIlmPolicy,
+      });
+    } catch (e) {
+      this.logger.error('Error in start phase');
+      this.logger.error(e.body.error);
+      throw e;
+    }
+  }
+
   public async addReport(report: Report): Promise<Report> {
     let index = report._index;
     if (!index) {

x-pack/plugins/reporting/server/plugin.ts

@@ -107,6 +107,9 @@ export class ReportingPlugin
         logger: this.logger,
       });
 
+      // Note: this must be called after ReportingCore.pluginStart
+      await store.start();
+
       this.logger.debug('Start complete');
     })().catch((e) => {
       this.logger.error(`Error in Reporting start, reporting may not function properly`);