From e35ecaa3785dd9521a1c144ee50b56084cc4c4f5 Mon Sep 17 00:00:00 2001 From: Ross Wolf <31489089+rw-access@users.noreply.github.com> Date: Wed, 14 Apr 2021 10:57:50 -0600 Subject: [PATCH] [Security] Adds pre-packaged rule updates through the "Prebuilt Security Detection Rules" Fleet integration (#96698) * Make the prepackaged rules functions async * Fix type for getPrepackagedRules mock * Install updates from saved objects & FS * Mock getLatestPrepackagedRules instead of getPrepackagedRules * Cleanup ruleAssetSavedObjectsClientFactory.all * Fix comment for "most recent version" * Switch to ruleMap.get() for less typescript errors * Remove unneeded constants * Fix SO.attributes sig and use custom validation --- .../rules/add_prepackaged_rules_route.test.ts | 2 +- .../rules/add_prepackaged_rules_route.ts | 11 +-- ...get_prepackaged_rules_status_route.test.ts | 2 +- .../get_prepackaged_rules_status_route.ts | 11 +-- .../rules/get_prepackaged_rules.test.ts | 6 +- .../rules/get_prepackaged_rules.ts | 68 ++++++++++++++++++- .../rules/rule_asset_saved_objects_client.ts | 47 +++++++++++++ .../lib/detection_engine/rules/types.ts | 13 ++++ 8 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts index 1195f9e5e1e967..026820a8f2ff76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.test.ts @@ -25,7 +25,7 @@ import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mo jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: (): AddPrepackagedRulesSchemaDecoded[] => { + getLatestPrepackagedRules: async (): Promise => { return [ { author: ['Elastic'], diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts index 8a8d6925b0e800..4f9bd7d0cfd6c5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/add_prepackaged_rules_route.ts @@ -25,12 +25,13 @@ import { SetupPlugins } from '../../../../plugin'; import { buildFrameworkRequest } from '../../../timeline/utils/common'; import { getIndexExists } from '../../index/get_index_exists'; -import { getPrepackagedRules } from '../../rules/get_prepackaged_rules'; +import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { installPrepackagedRules } from '../../rules/install_prepacked_rules'; import { updatePrepackagedRules } from '../../rules/update_prepacked_rules'; import { getRulesToInstall } from '../../rules/get_rules_to_install'; import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; import { transformError, buildSiemResponse } from '../utils'; import { AlertsClient } from '../../../../../../alerting/server'; @@ -110,7 +111,7 @@ export const createPrepackagedRules = async ( const savedObjectsClient = context.core.savedObjects.client; const exceptionsListClient = context.lists != null ? context.lists.getExceptionListClient() : exceptionsClient; - + const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient); if (!siemClient || !alertsClient) { throw new PrepackagedRulesError('', 404); } @@ -120,10 +121,10 @@ export const createPrepackagedRules = async ( await exceptionsListClient.createEndpointList(); } - const rulesFromFileSystem = getPrepackagedRules(); + const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient); const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); - const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); - const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); + const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules); + const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules); const signalsIndex = siemClient.getSignalsIndex(); if (rulesToInstall.length !== 0 || rulesToUpdate.length !== 0) { const signalsIndexExists = await getIndexExists(esClient.asCurrentUser, signalsIndex); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts index 9e843d463ab3e2..3c8321ee8eb9a5 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.test.ts @@ -23,7 +23,7 @@ import { jest.mock('../../rules/get_prepackaged_rules', () => { return { - getPrepackagedRules: () => { + getLatestPrepackagedRules: async () => { return [ { rule_id: 'rule-1', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts index c67f2cb6e9545f..33f9746fe9245c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/rules/get_prepackaged_rules_status_route.ts @@ -13,11 +13,12 @@ import { import type { SecuritySolutionPluginRouter } from '../../../../types'; import { DETECTION_ENGINE_PREPACKAGED_URL } from '../../../../../common/constants'; import { transformError, buildSiemResponse } from '../utils'; -import { getPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { getRulesToInstall } from '../../rules/get_rules_to_install'; import { getRulesToUpdate } from '../../rules/get_rules_to_update'; import { findRules } from '../../rules/find_rules'; +import { getLatestPrepackagedRules } from '../../rules/get_prepackaged_rules'; import { getExistingPrepackagedRules } from '../../rules/get_existing_prepackaged_rules'; +import { ruleAssetSavedObjectsClientFactory } from '../../rules/rule_asset_saved_objects_client'; import { buildFrameworkRequest } from '../../../timeline/utils/common'; import { ConfigType } from '../../../../config'; import { SetupPlugins } from '../../../../plugin'; @@ -40,15 +41,17 @@ export const getPrepackagedRulesStatusRoute = ( }, }, async (context, request, response) => { + const savedObjectsClient = context.core.savedObjects.client; const siemResponse = buildSiemResponse(response); const alertsClient = context.alerting?.getAlertsClient(); + const ruleAssetsClient = ruleAssetSavedObjectsClientFactory(savedObjectsClient); if (!alertsClient) { return siemResponse.error({ statusCode: 404 }); } try { - const rulesFromFileSystem = getPrepackagedRules(); + const latestPrepackagedRules = await getLatestPrepackagedRules(ruleAssetsClient); const customRules = await findRules({ alertsClient, perPage: 1, @@ -61,8 +64,8 @@ export const getPrepackagedRulesStatusRoute = ( const frameworkRequest = await buildFrameworkRequest(context, security, request); const prepackagedRules = await getExistingPrepackagedRules({ alertsClient }); - const rulesToInstall = getRulesToInstall(rulesFromFileSystem, prepackagedRules); - const rulesToUpdate = getRulesToUpdate(rulesFromFileSystem, prepackagedRules); + const rulesToInstall = getRulesToInstall(latestPrepackagedRules, prepackagedRules); + const rulesToUpdate = getRulesToUpdate(latestPrepackagedRules, prepackagedRules); const prepackagedTimelineStatus = await checkTimelinesStatus(frameworkRequest); const [validatedprepackagedTimelineStatus] = validate( prepackagedTimelineStatus, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts index 039bc8c1e2e497..2d92731dbbdfdf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.test.ts @@ -41,8 +41,10 @@ describe('get_existing_prepackaged_rules', () => { }); test('should throw an exception with a message having rule_id and name in it', () => { - // @ts-expect-error intentionally invalid argument - expect(() => getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }])).toThrow( + expect(() => + // @ts-expect-error intentionally invalid argument + getPrepackagedRules([{ name: 'rule name', rule_id: 'id-123' }]) + ).toThrow( 'name: "rule name", rule_id: "id-123" within the folder rules/prepackaged_rules is not a valid detection engine rule. Expect the system to not work with pre-packaged rules until this rule is fixed or the file is removed. Error is: Invalid value "undefined" supplied to "description",Invalid value "undefined" supplied to "risk_score",Invalid value "undefined" supplied to "severity",Invalid value "undefined" supplied to "type",Invalid value "undefined" supplied to "version", Full rule contents are:\n{\n "name": "rule name",\n "rule_id": "id-123"\n}' ); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts index 508238afcb6df9..b91557c6d7b1bb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_prepackaged_rules.ts @@ -19,6 +19,9 @@ import { BadRequestError } from '../errors/bad_request_error'; // TODO: convert rules files to TS and add explicit type definitions import { rawRules } from './prepackaged_rules'; +import { RuleAssetSavedObjectsClient } from './rule_asset_saved_objects_client'; +import { IRuleAssetSOAttributes } from './types'; +import { SavedObjectAttributes } from '../../../../../../../src/core/types'; /** * Validate the rules from the file system and throw any errors indicating to the developer @@ -52,7 +55,70 @@ export const validateAllPrepackagedRules = ( }); }; +/** + * Validate the rules from Saved Objects created by Fleet. + */ +export const validateAllRuleSavedObjects = ( + rules: Array +): AddPrepackagedRulesSchemaDecoded[] => { + return rules.map((rule) => { + const decoded = addPrepackagedRulesSchema.decode(rule); + const checked = exactCheck(rule, decoded); + + const onLeft = (errors: t.Errors): AddPrepackagedRulesSchemaDecoded => { + const ruleName = rule.name ? rule.name : '(rule name unknown)'; + const ruleId = rule.rule_id ? rule.rule_id : '(rule rule_id unknown)'; + throw new BadRequestError( + `name: "${ruleName}", rule_id: "${ruleId}" within the security-rule saved object ` + + `is not a valid detection engine rule. Expect the system ` + + `to not work with pre-packaged rules until this rule is fixed ` + + `or the file is removed. Error is: ${formatErrors( + errors + ).join()}, Full rule contents are:\n${JSON.stringify(rule, null, 2)}` + ); + }; + + const onRight = (schema: AddPrepackagedRulesSchema): AddPrepackagedRulesSchemaDecoded => { + return schema as AddPrepackagedRulesSchemaDecoded; + }; + return pipe(checked, fold(onLeft, onRight)); + }); +}; + +/** + * Retrieve and validate rules that were installed from Fleet as saved objects. + */ +export const getFleetInstalledRules = async ( + client: RuleAssetSavedObjectsClient +): Promise => { + const fleetResponse = await client.all(); + const fleetRules = fleetResponse.map((so) => so.attributes); + return validateAllRuleSavedObjects(fleetRules); +}; + export const getPrepackagedRules = ( // @ts-expect-error mock data is too loosely typed rules: AddPrepackagedRulesSchema[] = rawRules -): AddPrepackagedRulesSchemaDecoded[] => validateAllPrepackagedRules(rules); +): AddPrepackagedRulesSchemaDecoded[] => { + return validateAllPrepackagedRules(rules); +}; + +export const getLatestPrepackagedRules = async ( + client: RuleAssetSavedObjectsClient +): Promise => { + // build a map of the most recent version of each rule + const prepackaged = getPrepackagedRules(); + const ruleMap = new Map(prepackaged.map((r) => [r.rule_id, r])); + + // check the rules installed via fleet and create/update if the version is newer + const fleetRules = await getFleetInstalledRules(client); + const fleetUpdates = fleetRules.filter((r) => { + const rule = ruleMap.get(r.rule_id); + return rule == null || rule.version < r.version; + }); + + // add the new or updated rules to the map + fleetUpdates.forEach((r) => ruleMap.set(r.rule_id, r)); + + return Array.from(ruleMap.values()); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts new file mode 100644 index 00000000000000..ac0969dfc975d0 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/rule_asset_saved_objects_client.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SavedObjectsClientContract, + SavedObjectsFindOptions, + SavedObjectsFindResponse, +} from '../../../../../../../src/core/server'; +import { ruleAssetSavedObjectType } from '../rules/saved_object_mappings'; +import { IRuleAssetSavedObject } from '../rules/types'; + +const DEFAULT_PAGE_SIZE = 100; + +export interface RuleAssetSavedObjectsClient { + find: ( + options?: Omit + ) => Promise>; + all: () => Promise; +} + +export const ruleAssetSavedObjectsClientFactory = ( + savedObjectsClient: SavedObjectsClientContract +): RuleAssetSavedObjectsClient => { + return { + find: (options) => + savedObjectsClient.find({ + ...options, + type: ruleAssetSavedObjectType, + }), + all: async () => { + const finder = savedObjectsClient.createPointInTimeFinder({ + perPage: DEFAULT_PAGE_SIZE, + type: ruleAssetSavedObjectType, + }); + const responses: IRuleAssetSavedObject[] = []; + for await (const response of finder.find()) { + responses.push(...response.saved_objects.map((so) => so as IRuleAssetSavedObject)); + } + await finder.close(); + return responses; + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts index 2a87b008293216..2990a0f7280278 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts @@ -164,6 +164,19 @@ export interface IRuleStatusFindType { saved_objects: IRuleStatusSavedObject[]; } +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export interface IRuleAssetSOAttributes extends Record { + rule_id: string | null | undefined; + version: string | null | undefined; + name: string | null | undefined; +} + +export interface IRuleAssetSavedObject { + type: string; + id: string; + attributes: IRuleAssetSOAttributes & SavedObjectAttributes; +} + export interface HapiReadableStream extends Readable { hapi: { filename: string;