From e81c31c9fc7f0abbdb58c7f689fda20c863bd86b Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:07:44 +0300 Subject: [PATCH 1/5] fix index template for serverless --- .../cloud_security_posture/server/config.ts | 16 +++- .../server/create_indices/create_indices.ts | 85 +++++++++++++------ .../cloud_security_posture/server/plugin.ts | 7 +- 3 files changed, 77 insertions(+), 31 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/config.ts b/x-pack/plugins/cloud_security_posture/server/config.ts index 472f66c78be8c..cbc59ea2b79fc 100644 --- a/x-pack/plugins/cloud_security_posture/server/config.ts +++ b/x-pack/plugins/cloud_security_posture/server/config.ts @@ -5,13 +5,25 @@ * 2.0. */ -import { schema, type TypeOf } from '@kbn/config-schema'; +// import { schema, type TypeOf } from '@kbn/config-schema'; +// import { type TypeOf } from '@kbn/config-schema'; import type { PluginConfigDescriptor } from '@kbn/core/server'; +import { schema, offeringBasedSchema, type TypeOf } from '@kbn/config-schema'; + const configSchema = schema.object({ enabled: schema.boolean({ defaultValue: true }), + + // Setting only allowed in the Serverless offering + serverless: schema.object({ + enabled: offeringBasedSchema({ + serverless: schema.literal(true), + options: { defaultValue: schema.contextRef('serverless') }, + }), + }), }); -type CloudSecurityPostureConfig = TypeOf; + +export type CloudSecurityPostureConfig = TypeOf; export const config: PluginConfigDescriptor = { schema: configSchema, diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts index c269e201f4243..a8574268ea31d 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts @@ -4,10 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { errors } from '@elastic/elasticsearch'; -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; - import { BENCHMARK_SCORE_INDEX_DEFAULT_NS, BENCHMARK_SCORE_INDEX_PATTERN, @@ -20,8 +17,23 @@ import { latestFindingsPipelineIngestConfig, scorePipelineIngestConfig } from '. import { latestIndexConfigs } from './latest_indices'; import { IndexConfig, IndexTemplateParams } from './types'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { CloudSecurityPostureConfig } from '../config'; + +interface IndexTemplateSettings { + index: { + default_pipeline: string; + }; + lifecycle?: { name: string }; +} + // TODO: Add integration tests -export const initializeCspIndices = async (esClient: ElasticsearchClient, logger: Logger) => { +export const initializeCspIndices = async ( + esClient: ElasticsearchClient, + cloudSecurityPostureConfig: CloudSecurityPostureConfig, + logger: Logger +) => { await Promise.allSettled([ createPipelineIfNotExists(esClient, scorePipelineIngestConfig, logger), createPipelineIfNotExists(esClient, latestFindingsPipelineIngestConfig, logger), @@ -32,9 +44,14 @@ export const initializeCspIndices = async (esClient: ElasticsearchClient, logger createVulnerabilitiesLatestIndexPromise, createBenchmarkScoreIndexPromise, ] = await Promise.allSettled([ - createLatestIndex(esClient, logger, latestIndexConfigs.findings), - createLatestIndex(esClient, logger, latestIndexConfigs.vulnerabilities), - createBenchmarkScoreIndex(esClient, logger), + createLatestIndex(esClient, latestIndexConfigs.findings, cloudSecurityPostureConfig, logger), + createLatestIndex( + esClient, + latestIndexConfigs.vulnerabilities, + cloudSecurityPostureConfig, + logger + ), + createBenchmarkScoreIndex(esClient, cloudSecurityPostureConfig, logger), ]); if (createFindingsLatestIndexPromise.status === 'rejected') { @@ -48,25 +65,31 @@ export const initializeCspIndices = async (esClient: ElasticsearchClient, logger } }; -const createBenchmarkScoreIndex = async (esClient: ElasticsearchClient, logger: Logger) => { +const createBenchmarkScoreIndex = async ( + esClient: ElasticsearchClient, + cloudSecurityPostureConfig: CloudSecurityPostureConfig, + logger: Logger +) => { try { // Deletes old assets from previous versions as part of upgrade process const INDEX_TEMPLATE_V830 = 'cloud_security_posture.scores'; await deleteIndexTemplateSafe(esClient, logger, INDEX_TEMPLATE_V830); + const settings: IndexTemplateSettings = { + index: { + default_pipeline: latestFindingsPipelineIngestConfig.id, + }, + lifecycle: { name: '' }, + }; + if (cloudSecurityPostureConfig.serverless.enabled) delete settings.lifecycle; + // We always want to keep the index template updated await esClient.indices.putIndexTemplate({ name: BENCHMARK_SCORE_INDEX_TEMPLATE_NAME, index_patterns: BENCHMARK_SCORE_INDEX_PATTERN, template: { mappings: benchmarkScoreMapping, - settings: { - default_pipeline: scorePipelineIngestConfig.id, - // TODO: once we will convert the score index to datastream we will no longer override the ilm to be empty - lifecycle: { - name: '', - }, - }, + settings, }, _meta: { package: { @@ -98,8 +121,9 @@ const createBenchmarkScoreIndex = async (esClient: ElasticsearchClient, logger: const createLatestIndex = async ( esClient: ElasticsearchClient, - logger: Logger, - indexConfig: IndexConfig + indexConfig: IndexConfig, + cloudSecurityPostureConfig: CloudSecurityPostureConfig, + logger: Logger ) => { const { indexName, indexPattern, indexTemplateName, indexDefaultName } = indexConfig; try { @@ -121,7 +145,7 @@ const createLatestIndex = async ( }; // We always want to keep the index template updated - await updateIndexTemplate(esClient, logger, indexTemplateParams); + await updateIndexTemplate(esClient, indexTemplateParams, cloudSecurityPostureConfig, logger); const result = await createIndexSafe(esClient, logger, indexDefaultName); @@ -173,6 +197,7 @@ const createIndexSafe = async (esClient: ElasticsearchClient, logger: Logger, in }); if (!isLatestIndexExists) { + // esClient.indices.createDataStream await esClient.indices.create({ index, }); @@ -192,10 +217,21 @@ const createIndexSafe = async (esClient: ElasticsearchClient, logger: Logger, in const updateIndexTemplate = async ( esClient: ElasticsearchClient, - logger: Logger, - indexTemplateParams: IndexTemplateParams + indexTemplateParams: IndexTemplateParams, + cloudSecurityPostureConfig: CloudSecurityPostureConfig, + logger: Logger ) => { const { indexTemplateName, indexPattern, template, composedOf, _meta } = indexTemplateParams; + + const settings: IndexTemplateSettings = { + ...template?.settings, // nothing inside + index: { + default_pipeline: latestFindingsPipelineIngestConfig.id, + }, + lifecycle: { name: '' }, + }; + if (cloudSecurityPostureConfig.serverless.enabled) delete settings.lifecycle; + try { await esClient.indices.putIndexTemplate({ name: indexTemplateName, @@ -203,18 +239,13 @@ const updateIndexTemplate = async ( priority: 500, template: { mappings: template?.mappings, - settings: { - ...template?.settings, - default_pipeline: latestFindingsPipelineIngestConfig.id, - lifecycle: { - name: '', - }, - }, + settings, aliases: template?.aliases, }, _meta, composed_of: composedOf, }); + logger.info(`Updated index template successfully [Name: ${indexTemplateName}]`); } catch (e) { logger.error(`Failed to update index template [Name: ${indexTemplateName}]`); diff --git a/x-pack/plugins/cloud_security_posture/server/plugin.ts b/x-pack/plugins/cloud_security_posture/server/plugin.ts index 30ff887a354ca..8b77581efd39d 100755 --- a/x-pack/plugins/cloud_security_posture/server/plugin.ts +++ b/x-pack/plugins/cloud_security_posture/server/plugin.ts @@ -49,6 +49,7 @@ import { setupFindingsStatsTask, } from './tasks/findings_stats_task'; import { registerCspmUsageCollector } from './lib/telemetry/collectors/register'; +import { CloudSecurityPostureConfig } from './config'; export class CspPlugin implements @@ -60,6 +61,7 @@ export class CspPlugin > { private readonly logger: Logger; + private readonly config: CloudSecurityPostureConfig; private isCloudEnabled?: boolean; /** @@ -69,8 +71,9 @@ export class CspPlugin */ #isInitialized: boolean = false; - constructor(initializerContext: PluginInitializerContext) { + constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); + this.config = initializerContext.config.get(); } public setup( @@ -203,7 +206,7 @@ export class CspPlugin async initialize(core: CoreStart, taskManager: TaskManagerStartContract): Promise { this.logger.debug('initialize'); const esClient = core.elasticsearch.client.asInternalUser; - await initializeCspIndices(esClient, this.logger); + await initializeCspIndices(esClient, this.config, this.logger); await initializeCspTransforms(esClient, this.logger); await scheduleFindingsStatsTask(taskManager, this.logger); this.#isInitialized = true; From 04ab38751c880623d9f622e904f5fd00a83059e8 Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:12:15 +0300 Subject: [PATCH 2/5] debug --- .../cloud_security/cloud_security_metering_task_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts index aa35b9e870c2c..bb54cb676a1e6 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts @@ -8,7 +8,7 @@ import { cloudSecurityMetringCallback } from './cloud_security_metering'; import type { MetringTaskProperties } from '../types'; -const TASK_INTERVAL = 3600; // 1 hour +const TASK_INTERVAL = 120; // 1 hour export const cloudSecurityMetringTaskProperties: MetringTaskProperties = { taskType: 'cloud-security-usage-reporting-task', From f534d6035cba6148417ac6bda2c0e16911508cf7 Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:15:32 +0300 Subject: [PATCH 3/5] clear code --- x-pack/plugins/cloud_security_posture/server/config.ts | 3 --- .../server/create_indices/create_indices.ts | 1 - 2 files changed, 4 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/config.ts b/x-pack/plugins/cloud_security_posture/server/config.ts index cbc59ea2b79fc..735cdb3cbe00f 100644 --- a/x-pack/plugins/cloud_security_posture/server/config.ts +++ b/x-pack/plugins/cloud_security_posture/server/config.ts @@ -4,9 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - -// import { schema, type TypeOf } from '@kbn/config-schema'; -// import { type TypeOf } from '@kbn/config-schema'; import type { PluginConfigDescriptor } from '@kbn/core/server'; import { schema, offeringBasedSchema, type TypeOf } from '@kbn/config-schema'; diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts index a8574268ea31d..45e3b64a1744d 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts @@ -197,7 +197,6 @@ const createIndexSafe = async (esClient: ElasticsearchClient, logger: Logger, in }); if (!isLatestIndexExists) { - // esClient.indices.createDataStream await esClient.indices.create({ index, }); From a1d0177e84bd23b596748f633249b64b79014f79 Mon Sep 17 00:00:00 2001 From: Ido Cohen <90558359+CohenIdo@users.noreply.github.com> Date: Sun, 3 Sep 2023 10:37:29 +0300 Subject: [PATCH 4/5] clear code --- .../cloud_security/cloud_security_metering_task_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts index bb54cb676a1e6..aa35b9e870c2c 100644 --- a/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts +++ b/x-pack/plugins/security_solution_serverless/server/cloud_security/cloud_security_metering_task_config.ts @@ -8,7 +8,7 @@ import { cloudSecurityMetringCallback } from './cloud_security_metering'; import type { MetringTaskProperties } from '../types'; -const TASK_INTERVAL = 120; // 1 hour +const TASK_INTERVAL = 3600; // 1 hour export const cloudSecurityMetringTaskProperties: MetringTaskProperties = { taskType: 'cloud-security-usage-reporting-task', From 01fe707b3aa35f823c3f4d02fab443481dc3ad28 Mon Sep 17 00:00:00 2001 From: kibanamachine <42973632+kibanamachine@users.noreply.github.com> Date: Sun, 3 Sep 2023 08:13:24 +0000 Subject: [PATCH 5/5] [CI] Auto-commit changed files from 'node scripts/eslint --no-cache --fix' --- .../server/create_indices/create_indices.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts index 45e3b64a1744d..06e13248fa1c5 100644 --- a/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts +++ b/x-pack/plugins/cloud_security_posture/server/create_indices/create_indices.ts @@ -5,6 +5,8 @@ * 2.0. */ import { errors } from '@elastic/elasticsearch'; +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { BENCHMARK_SCORE_INDEX_DEFAULT_NS, BENCHMARK_SCORE_INDEX_PATTERN, @@ -17,8 +19,6 @@ import { latestFindingsPipelineIngestConfig, scorePipelineIngestConfig } from '. import { latestIndexConfigs } from './latest_indices'; import { IndexConfig, IndexTemplateParams } from './types'; -import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/types'; -import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import { CloudSecurityPostureConfig } from '../config'; interface IndexTemplateSettings {