From 3b511840f7b64c51b047d65dcacc798e9ae54b33 Mon Sep 17 00:00:00 2001 From: Lola Date: Tue, 28 Mar 2023 16:12:08 -0400 Subject: [PATCH] [Cloud Posture] update status API endpoint for vulnerability management findings (#153688) This PR: - Update API status endpoint for retrieving the Vulnerabilities integration status. - Implement logic to determine the status based on the following statuses( scenarios where the integration is not installed, installed but not configured, indexing in progress, indexing timed out, indexing complete, and indexing failed). - Add automated tests to ensure the API endpoint returns the correct status for each scenario. - Integrate the API endpoint into the system and test to ensure it's working properly. --- .../common/constants.ts | 9 ++ .../cloud_security_posture/common/types.ts | 3 +- .../server/lib/check_index_status.ts | 4 +- .../server/routes/status/status.test.ts | 144 ++++++++++++++++- .../server/routes/status/status.ts | 147 ++++++++++++++---- .../cloud_security_posture/server/types.ts | 16 ++ .../apis/cloud_security_posture/status.ts | 55 +++++-- 7 files changed, 329 insertions(+), 49 deletions(-) diff --git a/x-pack/plugins/cloud_security_posture/common/constants.ts b/x-pack/plugins/cloud_security_posture/common/constants.ts index 04e31f9e0b641..37453e1364f8d 100644 --- a/x-pack/plugins/cloud_security_posture/common/constants.ts +++ b/x-pack/plugins/cloud_security_posture/common/constants.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { PostureTypes } from './types'; + export const STATUS_ROUTE_PATH = '/internal/cloud_security_posture/status'; export const STATS_ROUTE_PATH = '/internal/cloud_security_posture/stats/{policy_template}'; export const BENCHMARKS_ROUTE_PATH = '/internal/cloud_security_posture/benchmarks'; @@ -85,3 +87,10 @@ export const SUPPORTED_CLOUDBEAT_INPUTS = [ CLOUDBEAT_VULN_MGMT_GCP, CLOUDBEAT_VULN_MGMT_AZURE, ] as const; + +export const POSTURE_TYPES: { [x: string]: PostureTypes } = { + [KSPM_POLICY_TEMPLATE]: KSPM_POLICY_TEMPLATE, + [CSPM_POLICY_TEMPLATE]: CSPM_POLICY_TEMPLATE, + [VULN_MGMT_POLICY_TEMPLATE]: VULN_MGMT_POLICY_TEMPLATE, + [POSTURE_TYPE_ALL]: POSTURE_TYPE_ALL, +} as const; diff --git a/x-pack/plugins/cloud_security_posture/common/types.ts b/x-pack/plugins/cloud_security_posture/common/types.ts index 83fd9a4b276c1..3552a663866e1 100644 --- a/x-pack/plugins/cloud_security_posture/common/types.ts +++ b/x-pack/plugins/cloud_security_posture/common/types.ts @@ -12,7 +12,7 @@ import { CspRuleTemplateMetadata } from './schemas/csp_rule_template_metadata'; export type Evaluation = 'passed' | 'failed' | 'NA'; -export type PostureTypes = 'cspm' | 'kspm' | 'all'; +export type PostureTypes = 'cspm' | 'kspm' | 'vuln_mgmt' | 'all'; /** number between 1-100 */ export type Score = number; @@ -85,6 +85,7 @@ export interface BaseCspSetupStatus { latestPackageVersion: string; cspm: BaseCspSetupBothPolicy; kspm: BaseCspSetupBothPolicy; + vuln_mgmt: BaseCspSetupBothPolicy; isPluginInitialized: boolean; } diff --git a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts index c8d876a7281da..47ab871dc9d21 100644 --- a/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts +++ b/x-pack/plugins/cloud_security_posture/server/lib/check_index_status.ts @@ -6,13 +6,13 @@ */ import { ElasticsearchClient, type Logger } from '@kbn/core/server'; -import { IndexStatus } from '../../common/types'; +import { IndexStatus, PostureTypes } from '../../common/types'; export const checkIndexStatus = async ( esClient: ElasticsearchClient, index: string, logger: Logger, - postureType: 'cspm' | 'kspm' | 'all' = 'all' + postureType: PostureTypes = 'all' ): Promise => { const query = postureType === 'all' diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts index 7f1345ced245f..5cf11f8d1f980 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.test.ts @@ -6,10 +6,10 @@ */ import { calculateCspStatusCode } from './status'; -import { CSPM_POLICY_TEMPLATE } from '../../../common/constants'; +import { CSPM_POLICY_TEMPLATE, VULN_MGMT_POLICY_TEMPLATE } from '../../../common/constants'; -describe('calculateCspStatusCode test', () => { - it('Verify status when there are no permission', async () => { +describe('calculateCspStatusCode for cspm', () => { + it('Verify status when there are no permission for cspm', async () => { const statusCode = calculateCspStatusCode( CSPM_POLICY_TEMPLATE, { @@ -145,3 +145,141 @@ describe('calculateCspStatusCode test', () => { expect(statusCode).toMatch('indexing'); }); }); + +describe('calculateCspStatusCode for vul_mgmt', () => { + it('Verify status when there are no permission for vul_mgmt', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'unprivileged', + findings: 'unprivileged', + score: 'unprivileged', + }, + 1, + 1, + 1, + ['cspm'] + ); + + expect(statusCode).toMatch('unprivileged'); + }); + + it('Verify status when there are no vul_mgmt findings, no healthy agents and no installed policy templates', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 0, + 0, + 0, + [] + ); + + expect(statusCode).toMatch('not-installed'); + }); + + it('Verify status when there are vul_mgmt findings and installed policies but no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are vul_mgmt findings ,installed policies and healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'not-empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexed'); + }); + + it('Verify status when there are no vul_mgmt findings ,installed policies and no healthy agents', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 0, + 10, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('not-deployed'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 9, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('waiting_for_results'); + }); + + it('Verify status when there are installed policies, healthy agents and no vul_mgmt findings and been more than 10 minutes', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'empty', + score: 'empty', + }, + 1, + 1, + 11, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('index-timeout'); + }); + + it('Verify status when there are installed policies, healthy agents past vul_mgmt findings but no recent findings', async () => { + const statusCode = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: 'empty', + findings: 'not-empty', + score: 'not-empty', + }, + 1, + 1, + 0, + [VULN_MGMT_POLICY_TEMPLATE] + ); + + expect(statusCode).toMatch('indexing'); + }); +}); diff --git a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts index 23578194422ee..e643741bc4320 100644 --- a/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts +++ b/x-pack/plugins/cloud_security_posture/server/routes/status/status.ts @@ -17,10 +17,14 @@ import { LATEST_FINDINGS_INDEX_DEFAULT_NS, FINDINGS_INDEX_PATTERN, BENCHMARK_SCORE_INDEX_DEFAULT_NS, + VULNERABILITIES_INDEX_PATTERN, KSPM_POLICY_TEMPLATE, CSPM_POLICY_TEMPLATE, + POSTURE_TYPES, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + VULN_MGMT_POLICY_TEMPLATE, } from '../../../common/constants'; -import type { CspApiRequestHandlerContext, CspRouter } from '../../types'; +import type { CspApiRequestHandlerContext, CspRouter, StatusResponseInfo } from '../../types'; import type { CspSetupStatus, CspStatusCode, @@ -72,7 +76,7 @@ export const calculateCspStatusCode = ( indicesStatus: { findingsLatest: IndexStatus; findings: IndexStatus; - score: IndexStatus; + score?: IndexStatus; }, installedCspPackagePolicies: number, healthyAgents: number, @@ -80,8 +84,7 @@ export const calculateCspStatusCode = ( installedPolicyTemplates: string[] ): CspStatusCode => { // We check privileges only for the relevant indices for our pages to appear - const postureTypeCheck = - postureType === CSPM_POLICY_TEMPLATE ? CSPM_POLICY_TEMPLATE : KSPM_POLICY_TEMPLATE; + const postureTypeCheck: PostureTypes = POSTURE_TYPES[postureType]; if (indicesStatus.findingsLatest === 'unprivileged' || indicesStatus.score === 'unprivileged') return 'unprivileged'; if (!installedPolicyTemplates.includes(postureTypeCheck)) return 'not-installed'; @@ -133,10 +136,13 @@ const getCspStatus = async ({ findingsLatestIndexStatusKspm, findingsIndexStatusKspm, scoreIndexStatusKspm, + vulnerabilitiesLatestIndexStatus, + vulnerabilitiesIndexStatus, installation, latestCspPackage, installedPackagePoliciesKspm, installedPackagePoliciesCspm, + installedPackagePoliciesVulnMgmt, installedPolicyTemplates, ] = await Promise.all([ checkIndexStatus(esClient.asCurrentUser, LATEST_FINDINGS_INDEX_DEFAULT_NS, logger), @@ -151,8 +157,22 @@ const getCspStatus = async ({ checkIndexStatus(esClient.asCurrentUser, FINDINGS_INDEX_PATTERN, logger, 'kspm'), checkIndexStatus(esClient.asCurrentUser, BENCHMARK_SCORE_INDEX_DEFAULT_NS, logger, 'kspm'), + checkIndexStatus( + esClient.asCurrentUser, + LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + checkIndexStatus( + esClient.asCurrentUser, + VULNERABILITIES_INDEX_PATTERN, + logger, + VULN_MGMT_POLICY_TEMPLATE + ), + packageService.asInternalUser.getInstallation(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), packageService.asInternalUser.fetchFindLatestPackage(CLOUD_SECURITY_POSTURE_PACKAGE_NAME), + getCspPackagePolicies( soClient, packagePolicyService, @@ -171,9 +191,17 @@ const getCspStatus = async ({ }, CSPM_POLICY_TEMPLATE ), + getCspPackagePolicies( + soClient, + packagePolicyService, + CLOUD_SECURITY_POSTURE_PACKAGE_NAME, + { + per_page: 10000, + }, + VULN_MGMT_POLICY_TEMPLATE + ), getInstalledPolicyTemplates(packagePolicyService, soClient), ]); - const healthyAgentsKspm = await getHealthyAgents( soClient, installedPackagePoliciesKspm.items, @@ -189,8 +217,18 @@ const getCspStatus = async ({ agentService, logger ); + + const healthyAgentsVulMgmt = await getHealthyAgents( + soClient, + installedPackagePoliciesVulnMgmt.items, + agentPolicyService, + agentService, + logger + ); const installedPackagePoliciesTotalKspm = installedPackagePoliciesKspm.total; const installedPackagePoliciesTotalCspm = installedPackagePoliciesCspm.total; + const installedPackagePoliciesTotalVulnMgmt = installedPackagePoliciesVulnMgmt.total; + const latestCspPackageVersion = latestCspPackage.version; const MIN_DATE = 0; @@ -207,6 +245,10 @@ const getCspStatus = async ({ index: BENCHMARK_SCORE_INDEX_DEFAULT_NS, status: scoreIndexStatus, }, + { + index: LATEST_VULNERABILITIES_INDEX_DEFAULT_NS, + status: vulnerabilitiesLatestIndexStatus, + }, ]; const statusCspm = calculateCspStatusCode( @@ -235,38 +277,38 @@ const getCspStatus = async ({ installedPolicyTemplates ); - if ((statusCspm && statusKspm) === 'not-installed') - return { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, - }, - indicesDetails, - latestPackageVersion: latestCspPackageVersion, - isPluginInitialized: isPluginInitialized(), - }; - - const response = { - cspm: { - status: statusCspm, - healthyAgents: healthyAgentsCspm, - installedPackagePolicies: installedPackagePoliciesTotalCspm, - }, - kspm: { - status: statusKspm, - healthyAgents: healthyAgentsKspm, - installedPackagePolicies: installedPackagePoliciesTotalKspm, + const statusVulnMgmt = calculateCspStatusCode( + VULN_MGMT_POLICY_TEMPLATE, + { + findingsLatest: vulnerabilitiesLatestIndexStatus, + findings: vulnerabilitiesIndexStatus, }, + installedPackagePoliciesTotalVulnMgmt, + healthyAgentsVulMgmt, + calculateDiffFromNowInMinutes(installation?.install_started_at || MIN_DATE), + installedPolicyTemplates + ); + + const statusResponseInfo = getStatusResponse({ + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, indicesDetails, - latestPackageVersion: latestCspPackageVersion, - installedPackageVersion: installation?.install_version, + latestCspPackageVersion, isPluginInitialized: isPluginInitialized(), + }); + + if ((statusCspm && statusKspm && statusVulnMgmt) === 'not-installed') return statusResponseInfo; + + const response = { + ...statusResponseInfo, + installedPackageVersion: installation?.install_version, }; assertResponse(response, logger); @@ -316,3 +358,40 @@ export const defineGetCspStatusRoute = (router: CspRouter): void => } } ); + +const getStatusResponse = (statusResponseInfo: StatusResponseInfo) => { + const { + statusCspm, + statusKspm, + statusVulnMgmt, + healthyAgentsCspm, + healthyAgentsKspm, + healthyAgentsVulMgmt, + installedPackagePoliciesTotalKspm, + installedPackagePoliciesTotalCspm, + installedPackagePoliciesTotalVulnMgmt, + indicesDetails, + latestCspPackageVersion, + isPluginInitialized, + }: StatusResponseInfo = statusResponseInfo; + return { + [CSPM_POLICY_TEMPLATE]: { + status: statusCspm, + healthyAgents: healthyAgentsCspm, + installedPackagePolicies: installedPackagePoliciesTotalCspm, + }, + [KSPM_POLICY_TEMPLATE]: { + status: statusKspm, + healthyAgents: healthyAgentsKspm, + installedPackagePolicies: installedPackagePoliciesTotalKspm, + }, + [VULN_MGMT_POLICY_TEMPLATE]: { + status: statusVulnMgmt, + healthyAgents: healthyAgentsVulMgmt, + installedPackagePolicies: installedPackagePoliciesTotalVulnMgmt, + }, + indicesDetails, + isPluginInitialized, + latestPackageVersion: latestCspPackageVersion, + }; +}; diff --git a/x-pack/plugins/cloud_security_posture/server/types.ts b/x-pack/plugins/cloud_security_posture/server/types.ts index 503a529f4a681..1b3c4fd840347 100644 --- a/x-pack/plugins/cloud_security_posture/server/types.ts +++ b/x-pack/plugins/cloud_security_posture/server/types.ts @@ -34,6 +34,7 @@ import type { import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import type { FleetStartContract, FleetRequestHandlerContext } from '@kbn/fleet-plugin/server'; import { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import { CspStatusCode, IndexDetails } from '../common/types'; // eslint-disable-next-line @typescript-eslint/no-empty-interface export interface CspServerPluginSetup {} @@ -97,3 +98,18 @@ export type CspRequestHandler< * @internal */ export type CspRouter = IRouter; + +export interface StatusResponseInfo { + statusCspm: CspStatusCode; + statusKspm: CspStatusCode; + statusVulnMgmt: CspStatusCode; + healthyAgentsCspm: number; + healthyAgentsKspm: number; + healthyAgentsVulMgmt: number; + installedPackagePoliciesTotalKspm: number; + installedPackagePoliciesTotalCspm: number; + installedPackagePoliciesTotalVulnMgmt: number; + indicesDetails: IndexDetails[]; + latestCspPackageVersion: string; + isPluginInitialized: boolean; +} diff --git a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts index 89ba33449ec16..ee8f851489e7b 100644 --- a/x-pack/test/api_integration/apis/cloud_security_posture/status.ts +++ b/x-pack/test/api_integration/apis/cloud_security_posture/status.ts @@ -74,9 +74,32 @@ export default function ({ getService }: FtrProviderContext) { expect(res.cspm.status).to.be('not-deployed'); expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-installed'); expect(res.cspm.healthyAgents).to.be(0); expect(res.cspm.installedPackagePolicies).to.be(1); }); + + it(`Should return not-deployed when vuln_mgmt is not installed`, async () => { + await createPackagePolicy( + supertest, + agentPolicyId, + 'vuln_mgmt', + 'cloudbeat/vuln_mgmt_aws', + 'aws', + 'vuln_mgmt' + ); + + const { body: res }: { body: CspSetupStatus } = await supertest + .get(`/internal/cloud_security_posture/status`) + .set('kbn-xsrf', 'xxxx') + .expect(200); + + expect(res.cspm.status).to.be('not-installed'); + expect(res.kspm.status).to.be('not-installed'); + expect(res.vuln_mgmt.status).to.be('not-deployed'); + expect(res.vuln_mgmt.healthyAgents).to.be(0); + expect(res.vuln_mgmt.installedPackagePolicies).to.be(1); + }); }); } @@ -88,6 +111,26 @@ async function createPackagePolicy( deployment: string, posture: string ) { + const version = posture === 'kspm' || posture === 'cspm' ? '1.2.8' : '1.3.0-preview2'; + const title = 'Security Posture Management'; + const streams = [ + { + enabled: false, + data_stream: { + type: 'logs', + dataset: 'cloud_security_posture.vulnerabilities', + }, + }, + ]; + + const inputTemplate = { + enabled: true, + type: input, + policy_template: policyTemplate, + }; + + const inputs = posture === 'vuln_mgmt' ? { ...inputTemplate, streams } : { ...inputTemplate }; + const { body: postPackageResponse } = await supertest .post(`/api/fleet/package_policies`) .set('kbn-xsrf', 'xxxx') @@ -98,17 +141,11 @@ async function createPackagePolicy( namespace: 'default', policy_id: agentPolicyId, enabled: true, - inputs: [ - { - enabled: true, - type: input, - policy_template: policyTemplate, - }, - ], + inputs: [inputs], package: { name: 'cloud_security_posture', - title: 'Kubernetes Security Posture Management', - version: '1.2.8', + title, + version, }, vars: { deployment: {