From 662fe7475738d90120342503e5a56016c2a5ee95 Mon Sep 17 00:00:00 2001 From: Jean-Louis Leysens Date: Tue, 25 May 2021 17:02:48 +0200 Subject: [PATCH] [Reporting] ILM policy for managing reporting indices (#100130) * wip; added logic for creating ILM policy at start up * added log when ilm policy is not created * added test for start function * updated ilm policy to not delete data * actually update jest snapshots and remove unused import * updated the ilm policy, removed the min_age for the hot phase * update jest snapshot * removed TODO comment * debug log -> info log Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../server/lib/store/report_ilm_policy.ts | 18 +++++ .../reporting/server/lib/store/store.test.ts | 39 +++++++++++ .../reporting/server/lib/store/store.ts | 66 +++++++++++++++---- x-pack/plugins/reporting/server/plugin.ts | 3 + 4 files changed, 114 insertions(+), 12 deletions(-) create mode 100644 x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts diff --git a/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts new file mode 100644 index 00000000000000..f4cd69a0331d7d --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/store/report_ilm_policy.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { PutLifecycleRequest } from '@elastic/elasticsearch/api/types'; + +export const reportingIlmPolicy: PutLifecycleRequest['body'] = { + policy: { + phases: { + hot: { + actions: {}, + }, + }, + }, +}; diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 7f96433fcc6ceb..fa35240dfc8fbd 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -7,6 +7,7 @@ import type { DeeplyMockedKeys } from '@kbn/utility-types/jest'; import { ElasticsearchClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; import { ReportingCore } from '../../'; import { createMockConfigSchema, @@ -16,6 +17,8 @@ import { import { Report, ReportDocument } from './report'; import { ReportingStore } from './store'; +const { createApiResponse } = elasticsearchServiceMock; + describe('ReportingStore', () => { const mockLogger = createMockLevelLogger(); let mockCore: ReportingCore; @@ -403,4 +406,40 @@ describe('ReportingStore', () => { ] `); }); + + describe('start', () => { + it('creates an ILM policy for managing reporting indices if there is not already one', async () => { + mockEsClient.ilm.getLifecycle.mockRejectedValueOnce(createApiResponse({ statusCode: 404 })); + mockEsClient.ilm.putLifecycle.mockResolvedValueOnce(createApiResponse()); + + const store = new ReportingStore(mockCore, mockLogger); + await store.start(); + + expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' }); + expect(mockEsClient.ilm.putLifecycle.mock.calls[0][0]).toMatchInlineSnapshot(` + Object { + "body": Object { + "policy": Object { + "phases": Object { + "hot": Object { + "actions": Object {}, + }, + }, + }, + }, + "policy": "kibana-reporting", + } + `); + }); + + it('does not create an ILM policy for managing reporting indices if one already exists', async () => { + mockEsClient.ilm.getLifecycle.mockResolvedValueOnce(createApiResponse()); + + const store = new ReportingStore(mockCore, mockLogger); + await store.start(); + + expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ policy: 'kibana-reporting' }); + expect(mockEsClient.ilm.putLifecycle).not.toHaveBeenCalled(); + }); + }); }); diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index fc7bd9c23d7693..9fb203fd5627ab 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -14,6 +14,7 @@ import { ReportTaskParams } from '../tasks'; import { indexTimestamp } from './index_timestamp'; import { mapping } from './mapping'; import { Report, ReportDocument, ReportSource } from './report'; +import { reportingIlmPolicy } from './report_ilm_policy'; /* * When searching for long-pending reports, we get a subset of fields @@ -71,19 +72,22 @@ export class ReportingStore { return exists; } - const indexSettings = { - number_of_shards: 1, - auto_expand_replicas: '0-1', - }; - const body = { - settings: indexSettings, - mappings: { - properties: mapping, - }, - }; - try { - await client.indices.create({ index: indexName, body }); + await client.indices.create({ + index: indexName, + body: { + settings: { + number_of_shards: 1, + auto_expand_replicas: '0-1', + lifecycle: { + name: this.ilmPolicyName, + }, + }, + mappings: { + properties: mapping, + }, + }, + }); return true; } catch (error) { @@ -130,6 +134,44 @@ export class ReportingStore { return client.indices.refresh({ index }); } + private readonly ilmPolicyName = 'kibana-reporting'; + + private async doesIlmPolicyExist(): Promise { + const client = await this.getClient(); + try { + await client.ilm.getLifecycle({ policy: this.ilmPolicyName }); + return true; + } catch (e) { + if (e.statusCode === 404) { + return false; + } + throw e; + } + } + + /** + * Function to be called during plugin start phase. This ensures the environment is correctly + * configured for storage of reports. + */ + public async start() { + const client = await this.getClient(); + try { + if (await this.doesIlmPolicyExist()) { + this.logger.debug(`Found ILM policy ${this.ilmPolicyName}; skipping creation.`); + return; + } + this.logger.info(`Creating ILM policy for managing reporting indices: ${this.ilmPolicyName}`); + await client.ilm.putLifecycle({ + policy: this.ilmPolicyName, + body: reportingIlmPolicy, + }); + } catch (e) { + this.logger.error('Error in start phase'); + this.logger.error(e.body.error); + throw e; + } + } + public async addReport(report: Report): Promise { let index = report._index; if (!index) { diff --git a/x-pack/plugins/reporting/server/plugin.ts b/x-pack/plugins/reporting/server/plugin.ts index fc52e10dd0cf94..efe1d9450bef31 100644 --- a/x-pack/plugins/reporting/server/plugin.ts +++ b/x-pack/plugins/reporting/server/plugin.ts @@ -107,6 +107,9 @@ export class ReportingPlugin logger: this.logger, }); + // Note: this must be called after ReportingCore.pluginStart + await store.start(); + this.logger.debug('Start complete'); })().catch((e) => { this.logger.error(`Error in Reporting start, reporting may not function properly`);